Skip to content

Instantly share code, notes, and snippets.

@tarekziade
Created January 9, 2017 21:03
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save tarekziade/be24439acd849fccdfcfef30f448c58e to your computer and use it in GitHub Desktop.
Save tarekziade/be24439acd849fccdfcfef30f448c58e to your computer and use it in GitHub Desktop.
Async Workers using Threads
import requests
import asyncio
import sys
from functools import partial
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
loop = asyncio.get_event_loop()
@contextmanager
def async_workers():
with ThreadPoolExecutor(max_workers=10) as executor:
yield partial(loop.run_in_executor, executor, requests.get)
async def get_pages(urls):
with async_workers() as get_page:
futures = [get_page(url) for url in urls]
for response in await asyncio.gather(*futures):
print(response.status_code)
if __name__ == '__main__':
urls = ['http://examples.com', 'http://example.org']
loop.run_until_complete(get_pages(urls))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment