Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
import asyncio
import os
import aiofiles
import aiohttp
# saved links from https://pybit.es/archives in urls
URLS = [u.rstrip() for u in open('urls', 'r').readlines()]
async def fetch(session, url):
async with session.get(url) as resp:
if resp.status == 200:
fname = os.path.basename(url)
async with aiofiles.open(f'downloads-fast/{fname}', mode='wb') as f:
print(f'Downloaded {fname} page')
await f.write(await resp.read())
await f.close()
async def main():
async with aiohttp.ClientSession() as session:
tasks = [asyncio.create_task(fetch(session, url)) for url in URLS]
await asyncio.gather(*tasks)
asyncio.run(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment