Skip to content

Instantly share code, notes, and snippets.

@logston
Last active July 21, 2016 01:21
Show Gist options
  • Save logston/dfc9927a75a4074e8b3a0bbda60c7aa8 to your computer and use it in GitHub Desktop.
Save logston/dfc9927a75a4074e8b3a0bbda60c7aa8 to your computer and use it in GitHub Desktop.
A process pool that spins up thread pools that spins up asyncio loops.
"""
A process pool that spins up thread pools that spins up asyncio loops.
"""
import asyncio
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, wait
import random
PROCESS_WORKERS = 2
THREAD_WORKERS = 2
CORO_WORKERS = 10
async def print_status(loop, process_id, thread_id, worker_id):
while True:
sleep_time = random.random()
print('P', process_id, 'T', thread_id, 'A', worker_id, 'SLEEP', sleep_time, flush=True)
await asyncio.sleep(sleep_time, loop=loop)
def start_loop(process_id, thread_id):
print('Starting loop in thread', thread_id, 'and process', process_id)
loop = asyncio.new_event_loop()
tasks = [print_status(loop, process_id, thread_id, i) for i in range(CORO_WORKERS)]
tasks = [asyncio.ensure_future(task, loop=loop) for task in tasks]
loop.run_until_complete(asyncio.wait(tasks, loop=loop))
loop.close()
def start_threads(process_id):
print('Starting threads in process', process_id)
executor = ThreadPoolExecutor(max_workers=THREAD_WORKERS)
futures = []
for thread_id in range(THREAD_WORKERS):
futures.append(executor.submit(start_loop, process_id, thread_id))
wait(futures)
def start_processes():
executor = ProcessPoolExecutor(max_workers=PROCESS_WORKERS)
for process_id in range(PROCESS_WORKERS):
executor.submit(start_threads, process_id)
start_processes()
➜ ~ python loops_in_threads_in_procs.py
Starting threads in process 0
Starting loop in thread 0 and process 0
Starting threads in process 1
Starting loop in thread 0 and process 1
Starting loop in thread 1 and process 0
P 0 T 0 A 0 SLEEP 0.32496261607700283
P 1 T 0 A 0 SLEEP 0.6932477902954257
P 0 T 1 A 0 SLEEP 0.8125339101769086
P 0 T 1 A 1 SLEEP 0.8954271206383917
Starting loop in thread 1 and process 1
P 0 T 0 A 1 SLEEP 0.5214449561390824
P 1 T 0 A 1 SLEEP 0.7626271576587527
P 0 T 1 A 2 SLEEP 0.3822534430044633
P 0 T 0 A 2 SLEEP 0.28973768263403943
P 0 T 1 A 3 SLEEP 0.41749142526788474
P 0 T 0 A 3 SLEEP 0.5306875977432969
P 1 T 1 A 0 SLEEP 0.08529706813629001
P 0 T 1 A 4 SLEEP 0.9616402661295792
P 1 T 0 A 2 SLEEP 0.893155293191048
P 1 T 1 A 1 SLEEP 0.4846477160193303
P 0 T 0 A 4 SLEEP 0.8680217570201527
P 0 T 1 A 5 SLEEP 0.6120150441201887
P 1 T 1 A 2 SLEEP 0.7836633648034044
P 1 T 0 A 3 SLEEP 0.526837822865518
P 0 T 0 A 5 SLEEP 0.6219654472699936
P 1 T 1 A 3 SLEEP 0.29472064724911673
P 1 T 0 A 4 SLEEP 0.3270487448377163
P 0 T 1 A 6 SLEEP 0.32589563757950535
P 1 T 0 A 5 SLEEP 0.13297904202652744
P 0 T 0 A 6 SLEEP 0.709034063527328
P 0 T 1 A 7 SLEEP 0.16193826737654016
P 1 T 1 A 4 SLEEP 0.8103037817779705
P 0 T 0 A 7 SLEEP 0.16671847745810986
P 1 T 0 A 6 SLEEP 0.20653418394289902
P 0 T 1 A 8 SLEEP 0.5416933013912
P 0 T 0 A 8 SLEEP 0.12957002726026012
P 1 T 1 A 5 SLEEP 0.5175189312606152
P 1 T 0 A 7 SLEEP 0.127762772525304
P 0 T 1 A 9 SLEEP 0.12215144266004441
P 0 T 0 A 9 SLEEP 0.8067586413378853
P 1 T 1 A 6 SLEEP 0.5787983137059023
P 1 T 0 A 8 SLEEP 0.13241201415684944
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment