Trial of ProcessPoolExecutor

executor_test.py

import time
from concurrent.futures import ProcessPoolExecutor, as_completed
import random

import jsonlines as jl


class MyExecutor:
    def write(self, text: str):
        time.sleep(random.randint(0, 2))
        return {"now": time.time(), "text": text}

    def run(self):
        print("start")
        with ProcessPoolExecutor(max_workers=4) as executor, jl.open(
            "executor_test.jsonl", "w"
        ) as writer:
            futures = [executor.submit(self.write, f"test{i}") for i in range(10)]
            print("dispatched")
            for future in as_completed(futures):
                writer.write(future.result())
        print("end")


if __name__ == "__main__":
    MyExecutor().run()