crawl-async.py

import asyncio
import os

import aiofiles
import aiohttp

# saved links from https://pybit.es/archives in urls
URLS = [u.rstrip() for u in open('urls', 'r').readlines()]

async def fetch(session, url):
    async with session.get(url) as resp:
        if resp.status == 200:
            fname = os.path.basename(url)
            async with aiofiles.open(f'downloads-fast/{fname}', mode='wb') as f:
                print(f'Downloaded {fname} page')
                await f.write(await resp.read())
                await f.close()


async def main():
    async with aiohttp.ClientSession() as session:
        tasks = [asyncio.create_task(fetch(session, url)) for url in URLS]
        await asyncio.gather(*tasks)


asyncio.run(main())