Skip to content

Instantly share code, notes, and snippets.

@fabiocerqueira
Last active October 20, 2019 12:23
Show Gist options
  • Save fabiocerqueira/77556f44f65afca65e6b28539f31127d to your computer and use it in GitHub Desktop.
Save fabiocerqueira/77556f44f65afca65e6b28539f31127d to your computer and use it in GitHub Desktop.
sync running in a process pool executor
import asyncio
from pprint import pprint
from concurrent.futures import ProcessPoolExecutor
import requests
def io_block_socket_call(url):
resp = url, requests.get(url).status_code == 200
return resp
async def io_block_in_executor(url, task_id):
loop = asyncio.get_event_loop()
print("noblock pre", url, task_id)
resp = await loop.run_in_executor(None, io_block_socket_call, url)
print("noblock pos", url, task_id)
return resp
async def main():
loop = asyncio.get_running_loop()
executor = ProcessPoolExecutor(max_workers=4)
loop.set_default_executor(executor)
urls = [
"https://www.google.com",
"https://www.pylestras.org",
"https://www.python.org",
"https://www.globo.com",
"https://www.uol.com.br",
"https://stackoverflow.com",
]
tasks = [io_block_in_executor(url, task_id) for task_id, url in enumerate(urls)]
result = await asyncio.gather(*tasks)
print("Results:")
pprint(result)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment