Skip to content

Instantly share code, notes, and snippets.

@lc-thomas
Created September 28, 2021 16:17
Show Gist options
  • Save lc-thomas/0853a7369784073a92a20ff61e7c0052 to your computer and use it in GitHub Desktop.
Save lc-thomas/0853a7369784073a92a20ff61e7c0052 to your computer and use it in GitHub Desktop.
Python simple multiprocessing + asyncio - one event loop per process
from multiprocessing import Process
import asyncio as aio
import coloredlogs, logging
import time
PROCESSES = 2
COROUTINES_PER_PROCESS = 5
class Multitasker:
def __init__(self):
logger.info('Running multitasker')
def run(self):
logger.info(f'Spawning {PROCESSES} processes')
processes = [Process(target=self.process, daemon=True) for p in range(PROCESSES)]
[p.start() for p in processes]
[p.join() for p in processes]
def process(self):
logger.info(f" - process started, starting {COROUTINES_PER_PROCESS} coroutines")
loop = aio.new_event_loop()
aio.set_event_loop(loop)
tasks = [aio.ensure_future(self.main_coroutine(n_coroutine)) for n_coroutine in range(COROUTINES_PER_PROCESS)]
loop.run_until_complete(aio.wait(tasks))
async def main_coroutine(self, n_coroutine):
logger.info(f' - running coroutine {n_coroutine} waiting {n_coroutine + 1} secs')
await aio.sleep(n_coroutine + 1)
logger.info(f' - done')
if __name__ == '__main__':
logger = logging.getLogger('log')
logger.setLevel(logging.DEBUG)
coloredlogs.install(logger=logger, fmt='%(asctime)s [%(process)d] %(levelname)s %(message)s')
mt = Multitasker()
mt.run()
@lc-thomas
Copy link
Author

Output :

multitasker | 2021-09-28 16:14:25 [1] INFO Running multitasker
multitasker | 2021-09-28 16:14:25 [1] INFO Spawning 2 processes
multitasker | 2021-09-28 16:14:25 [7] INFO - process started, starting 5 coroutines
multitasker | 2021-09-28 16:14:25 [8] INFO - process started, starting 5 coroutines
multitasker | 2021-09-28 16:14:25 [7] INFO - running coroutine 0 waiting 1 secs
multitasker | 2021-09-28 16:14:25 [7] INFO - running coroutine 1 waiting 2 secs
multitasker | 2021-09-28 16:14:25 [7] INFO - running coroutine 2 waiting 3 secs
multitasker | 2021-09-28 16:14:25 [7] INFO - running coroutine 3 waiting 4 secs
multitasker | 2021-09-28 16:14:25 [7] INFO - running coroutine 4 waiting 5 secs
multitasker | 2021-09-28 16:14:25 [8] INFO - running coroutine 0 waiting 1 secs
multitasker | 2021-09-28 16:14:25 [8] INFO - running coroutine 1 waiting 2 secs
multitasker | 2021-09-28 16:14:25 [8] INFO - running coroutine 2 waiting 3 secs
multitasker | 2021-09-28 16:14:25 [8] INFO - running coroutine 3 waiting 4 secs
multitasker | 2021-09-28 16:14:25 [8] INFO - running coroutine 4 waiting 5 secs
multitasker | 2021-09-28 16:14:26 [8] INFO - done
multitasker | 2021-09-28 16:14:26 [7] INFO - done
multitasker | 2021-09-28 16:14:27 [8] INFO - done
multitasker | 2021-09-28 16:14:27 [7] INFO - done
multitasker | 2021-09-28 16:14:28 [7] INFO - done
multitasker | 2021-09-28 16:14:28 [8] INFO - done
multitasker | 2021-09-28 16:14:29 [7] INFO - done
multitasker | 2021-09-28 16:14:29 [8] INFO - done
multitasker | 2021-09-28 16:14:30 [7] INFO - done
multitasker | 2021-09-28 16:14:30 [8] INFO - done

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment