Script to download files in a async way, using Python asyncio

async_download_files.py

import os
import asyncio
import aiohttp  # pip install aiohttp
import aiofiles  # pip install aiofiles

REPORTS_FOLDER = "reports"
FILES_PATH = os.path.join(REPORTS_FOLDER, "files")


def download_files_from_report(urls):
    os.makedirs(FILES_PATH, exist_ok=True)
    sema = asyncio.BoundedSemaphore(5)

    async def fetch_file(url):
        fname = url.split("/")[-1]
        async with sema, aiohttp.ClientSession() as session:
            async with session.get(url) as resp:
                assert resp.status == 200
                data = await resp.read()

        async with aiofiles.open(
            os.path.join(FILES_PATH, fname), "wb"
        ) as outfile:
            await outfile.write(data)

    loop = asyncio.get_event_loop()
    tasks = [loop.create_task(fetch_file(url)) for url in urls]
    loop.run_until_complete(asyncio.wait(tasks))
    loop.close()

Use aiofile (not aiofiles) for better performances. aiofiles is using multithreading behind the scenes while aiofile uses platform native api on Linux/MacOS, thus being real async

Brilliant code thanks

Great piece of code!