A process pool that spins up thread pools that spins up asyncio loops.

loops_in_threads_in_processes.py

"""
A process pool that spins up thread pools that spins up asyncio loops. 
"""
import asyncio
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, wait
import random


PROCESS_WORKERS = 2
THREAD_WORKERS = 2
CORO_WORKERS = 10


async def print_status(loop, process_id, thread_id, worker_id):
    while True:
        sleep_time = random.random()
        print('P', process_id, 'T', thread_id, 'A', worker_id, 'SLEEP', sleep_time, flush=True)
        await asyncio.sleep(sleep_time, loop=loop)


def start_loop(process_id, thread_id):
    print('Starting loop in thread', thread_id, 'and process', process_id)
    loop = asyncio.new_event_loop()
    tasks = [print_status(loop, process_id, thread_id, i) for i in range(CORO_WORKERS)]
    tasks = [asyncio.ensure_future(task, loop=loop) for task in tasks]
    loop.run_until_complete(asyncio.wait(tasks, loop=loop))
    loop.close()


def start_threads(process_id):
    print('Starting threads in process', process_id)
    executor = ThreadPoolExecutor(max_workers=THREAD_WORKERS)
    futures = []
    for thread_id in range(THREAD_WORKERS):
        futures.append(executor.submit(start_loop, process_id, thread_id))
    wait(futures)


def start_processes():
    executor = ProcessPoolExecutor(max_workers=PROCESS_WORKERS)
    for process_id in range(PROCESS_WORKERS):
        executor.submit(start_threads, process_id)


start_processes()

sample_output.txt

➜  ~ python loops_in_threads_in_procs.py
Starting threads in process 0
Starting loop in thread 0 and process 0
Starting threads in process 1
Starting loop in thread 0 and process 1
Starting loop in thread 1 and process 0
P 0 T 0 A 0 SLEEP 0.32496261607700283
P 1 T 0 A 0 SLEEP 0.6932477902954257
P 0 T 1 A 0 SLEEP 0.8125339101769086
P 0 T 1 A 1 SLEEP 0.8954271206383917
Starting loop in thread 1 and process 1
P 0 T 0 A 1 SLEEP 0.5214449561390824
P 1 T 0 A 1 SLEEP 0.7626271576587527
P 0 T 1 A 2 SLEEP 0.3822534430044633
P 0 T 0 A 2 SLEEP 0.28973768263403943
P 0 T 1 A 3 SLEEP 0.41749142526788474
P 0 T 0 A 3 SLEEP 0.5306875977432969
P 1 T 1 A 0 SLEEP 0.08529706813629001
P 0 T 1 A 4 SLEEP 0.9616402661295792
P 1 T 0 A 2 SLEEP 0.893155293191048
P 1 T 1 A 1 SLEEP 0.4846477160193303
P 0 T 0 A 4 SLEEP 0.8680217570201527
P 0 T 1 A 5 SLEEP 0.6120150441201887
P 1 T 1 A 2 SLEEP 0.7836633648034044
P 1 T 0 A 3 SLEEP 0.526837822865518
P 0 T 0 A 5 SLEEP 0.6219654472699936
P 1 T 1 A 3 SLEEP 0.29472064724911673
P 1 T 0 A 4 SLEEP 0.3270487448377163
P 0 T 1 A 6 SLEEP 0.32589563757950535
P 1 T 0 A 5 SLEEP 0.13297904202652744
P 0 T 0 A 6 SLEEP 0.709034063527328
P 0 T 1 A 7 SLEEP 0.16193826737654016
P 1 T 1 A 4 SLEEP 0.8103037817779705
P 0 T 0 A 7 SLEEP 0.16671847745810986
P 1 T 0 A 6 SLEEP 0.20653418394289902
P 0 T 1 A 8 SLEEP 0.5416933013912
P 0 T 0 A 8 SLEEP 0.12957002726026012
P 1 T 1 A 5 SLEEP 0.5175189312606152
P 1 T 0 A 7 SLEEP 0.127762772525304
P 0 T 1 A 9 SLEEP 0.12215144266004441
P 0 T 0 A 9 SLEEP 0.8067586413378853
P 1 T 1 A 6 SLEEP 0.5787983137059023
P 1 T 0 A 8 SLEEP 0.13241201415684944