Skip to content

Instantly share code, notes, and snippets.

@novelview9
Created May 16, 2022 17:56
Show Gist options
  • Save novelview9/763ffefe5f1cbb096c7c8396b30b1e0e to your computer and use it in GitHub Desktop.
Save novelview9/763ffefe5f1cbb096c7c8396b30b1e0e to your computer and use it in GitHub Desktop.
python multi image file download
import nest_asyncio # pip install nest_asyncio (for ipython)
nest_asyncio.apply()
import os
import asyncio
import aiohttp # pip install aiohttp
import aiofiles # pip install aiofiles
REPORTS_FOLDER = "reports"
FILES_PATH = os.path.join(REPORTS_FOLDER, "files")
def download_files_from_report(urls):
os.makedirs(FILES_PATH, exist_ok=True)
sema = asyncio.BoundedSemaphore(5)
async def fetch_file(url):
fname = url.split("/")[-1]
async with sema, aiohttp.ClientSession() as session:
async with session.get(url) as resp:
assert resp.status == 200
data = await resp.read()
async with aiofiles.open(
os.path.join(FILES_PATH, fname), "wb"
) as outfile:
await outfile.write(data)
loop = asyncio.get_event_loop()
tasks = [loop.create_task(fetch_file(url)) for url in urls]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
urls = ['https://test.png', 'https://test2.png']
download_files_from_report(urls)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment