yield_results_asynchronously.py

# Yield results asynchronously in python
# https://stackoverflow.com/questions/23351332/yield-results-asynchronously-in-python-using-multiprocessing-or-twisted
# https://medium.com/@apbetahouse45/asynchronous-web-scraping-in-python-using-concurrent-module-a5ca1b7f82e4
# https://stackoverflow.com/questions/22445054/asyncio-yield-from-concurrent-futures-future-of-an-executor
# https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Executor.submit

import time 
import multiprocessing 
from concurrent.futures import ProcessPoolExecutor 
 
 
def heavy_memory_task(x): 
    l = [1,2,3,4,5] 
    time.sleep(x) 
    return [i * x for i in l] 
 
def evaluate_things_in_parallel(): 
    with ProcessPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor: 
        futures = [executor.submit(heavy_memory_task, num) for num in [2, 3, 1, 4]] 
        # for result in executor.map(heavy_memory_task, [2, 3, 1, 4]): 
        for result in concurrent.futures.as_completed(futures): 
            yield result.result() 
 
r = evaluate_things_in_parallel() 
for i in r: 
    print(i) 
                                                                                                                                                
[1, 2, 3, 4, 5]
[2, 4, 6, 8, 10]
[3, 6, 9, 12, 15]
[4, 8, 12, 16, 20]