asyncio_requests_example.py

#!/usr/bin/env python3

# ~~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~
# asyncio + requests example - (c)oded 2018 Jon, bitquark.io
# ~~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~

import asyncio
import requests
import concurrent.futures


# Number of workers and per-request timeout
settings = { 'workers': 12, 'timeout': 2 }

# URLs to fetch
urls = ['https://news.ycombinator.com/' for _ in range(0, 16)]


def fetch_url(url, timeout):
    """ Fetch the given URL """

    # Fetch the URL with requests, handling timeouts appropriately
    try:
        response = requests.get(url, verify=False, timeout=timeout)
        return (response.status_code, response.elapsed.total_seconds(), url)
    except requests.exceptions.ReadTimeout:
        return ('timeout', '-', url)


def process_fetch_result(result):
    """ Process results from fetch_url() """

    # Output the fetch result
    print('{:<12} {:<16} {}'.format(*result))


async def fetch_all_urls(loop, executor, urls, timeout):
    """ Coroutine to fetch all specified URLs """

    # Build tasks using the thread pool executor
    tasks = [loop.run_in_executor(executor, fetch_url, url, timeout) for url in urls]

    # Like asyncio.gather(*tasks) but handles results as they come in
    while tasks:
        done, tasks = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
        for task in done:
            process_fetch_result(task.result())


def main():
    """ Main code block """

    # Set up the executor
    executor = concurrent.futures.ThreadPoolExecutor(settings['workers'])

    # Kick off the loop
    loop = asyncio.get_event_loop()
    loop.run_until_complete(fetch_all_urls(loop, executor, urls, settings['timeout']))


# Go!
if __name__ == '__main__':
    main()