Skip to content

Instantly share code, notes, and snippets.

@alexzanderr
Forked from khardix/download.py
Created February 6, 2021 08:06
Show Gist options
  • Save alexzanderr/9b159a1bb687364c596451aad7c5f9e3 to your computer and use it in GitHub Desktop.
Save alexzanderr/9b159a1bb687364c596451aad7c5f9e3 to your computer and use it in GitHub Desktop.
Python AsyncIO/aiohttp downloader with progressbars
#!/usr/bin/env python3.6
import asyncio
from contextlib import closing
import aiohttp
import tqdm
async def download(session, url, progress_queue):
async with session.get(url) as response:
target = url.rpartition('/')[-1]
size = int(response.headers.get('content-length', 0)) or None
position = await progress_queue.get()
progressbar = tqdm.tqdm(
desc=target, total=size, position=position, leave=False,
)
with open(target, mode='wb') as f, progressbar:
async for chunk in response.content.iter_chunked(512):
f.write(chunk)
progressbar.update(len(chunk))
await progress_queue.put(position)
return target
async def main(loop):
with open('urls.txt') as f:
urls = [url.strip() for url in f]
progress_queue = asyncio.Queue(loop=loop)
for pos in range(5):
progress_queue.put_nowait(pos)
async with aiohttp.ClientSession(loop=loop) as session:
tasks = [download(session, url, progress_queue) for url in urls]
return await asyncio.gather(*tasks)
with closing(asyncio.get_event_loop()) as loop:
for tgt in loop.run_until_complete(main(loop)):
print(tgt)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment