Fetching via queue

async_fetch.py

import asyncio
import aiohttp


async def fetch(session, url):
    async with session.get(url) as response:
        return await response.text()


async def consumer(queue, session):
    while True:
        url = await queue.get()
        if url is None:
            break
        try:
            return await fetch(session, url)
        except aiohttp.ClientError as e:
            print(e)

        queue.task_done()


async def main():
    queue = asyncio.Queue()
    urls = [
        'http://python.org',
        'https://google.com',
        'http://yifei.me'
    ]
    for url in urls:
        queue.put_nowait(url)
    queue.put_nowait(None)
    async with aiohttp.ClientSession() as session:
        tasks = [consumer(queue, session) for _ in range(3)]
        htmls = await asyncio.gather(*tasks, return_exceptions=True)

        for html in htmls:
            print(html[:100])


if __name__ == "__main__":
    asyncio.run(main())