Thread overhead

test.py

from subprocess import Popen
from concurrent.futures import ThreadPoolExecutor
from os import waitpid
import time
import asyncio


start = time.time()
for i in range(3000):
    p1 = Popen(['/bin/true'])
    p1.wait()
print('Popen(1):', time.time() - start)


start = time.time()
ongoing = []
i = 0
while i < 3000:
    while len(ongoing) < 2:
        ongoing.append(Popen(['/bin/true']))
    waitpid(-1, 0)
    for pos, p in enumerate(ongoing):
        if p.poll() is not None:
            del ongoing[pos]
            i += 1
            break
print('Popen(2):', time.time() - start)


start = time.time()
ongoing = []
i = 0
while i < 3000:
    while len(ongoing) < 3:
        ongoing.append(Popen(['/bin/true']))
    waitpid(-1, 0)
    for pos, p in enumerate(ongoing):
        if p.poll() is not None:
            del ongoing[pos]
            i += 1
            break
print('Popen(3):', time.time() - start)


start = time.time()
with ThreadPoolExecutor(1) as e:
    e.map(lambda i: Popen('/bin/true').wait(), range(3000))
print('Thred(1):', time.time() - start)


start = time.time()
with ThreadPoolExecutor(2) as e:
    e.map(lambda i: Popen('/bin/true').wait(), range(3000))
print('Thred(2):', time.time() - start)


start = time.time()
with ThreadPoolExecutor(3) as e:
    e.map(lambda i: Popen('/bin/true').wait(), range(3000))
print('Thred(3):', time.time() - start)


start = time.time()
async def async1():
    for i in range(3000):
        p = await asyncio.create_subprocess_exec('/bin/true')
        await p.wait()
asyncio.get_event_loop().run_until_complete(async1())
print('Async(1):', time.time() - start)


start = time.time()
async def async2():
    async def work():
        p = await asyncio.create_subprocess_exec('/bin/true')
        await p.wait()
    pending = set()
    i = 0
    while i < 3000:
        while len(pending) < 2:
            pending.add(work())
        done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
        i += len(done)
asyncio.get_event_loop().run_until_complete(async2())
print('Async(2):', time.time() - start)


start = time.time()
async def async3():
    async def work():
        p = await asyncio.create_subprocess_exec('/bin/true')
        await p.wait()
    pending = set()
    i = 0
    while i < 3000:
        while len(pending) < 3:
            pending.add(work())
        done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
        i += len(done)
asyncio.get_event_loop().run_until_complete(async3())
print('Async(3):', time.time() - start)