Skip to content

Instantly share code, notes, and snippets.

@pybites
Created February 9, 2021 16:43
Show Gist options
  • Save pybites/6e602ee43599f013f211edf73dde4e69 to your computer and use it in GitHub Desktop.
Save pybites/6e602ee43599f013f211edf73dde4e69 to your computer and use it in GitHub Desktop.
import asyncio
import os
import aiofiles
import aiohttp
# saved links from https://pybit.es/archives in urls
URLS = [u.rstrip() for u in open('urls', 'r').readlines()]
async def fetch(session, url):
async with session.get(url) as resp:
if resp.status == 200:
fname = os.path.basename(url)
async with aiofiles.open(f'downloads-fast/{fname}', mode='wb') as f:
print(f'Downloaded {fname} page')
await f.write(await resp.read())
await f.close()
async def main():
async with aiohttp.ClientSession() as session:
tasks = [asyncio.create_task(fetch(session, url)) for url in URLS]
await asyncio.gather(*tasks)
asyncio.run(main())
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment