Skip to content

Instantly share code, notes, and snippets.

@brouberol
Last active September 14, 2021 13:10
Show Gist options
  • Save brouberol/2b8e77b25b4371df5b73876fc1974565 to your computer and use it in GitHub Desktop.
Save brouberol/2b8e77b25b4371df5b73876fc1974565 to your computer and use it in GitHub Desktop.
diff --git "a/t\303\251l\303\251chargeur-cheerz.py" "b/t\303\251l\303\251chargeur-cheerz.py"
index 871a79f..b8db037 100755
--- "a/t\303\251l\303\251chargeur-cheerz.py"
+++ "b/t\303\251l\303\251chargeur-cheerz.py"
@@ -1,6 +1,8 @@
#!/usr/bin/env python3
import json
+import aiohttp
+
from argparse import ArgumentParser
from asyncio import Task, create_task, run, wait
from dataclasses import dataclass
@@ -44,19 +46,28 @@ class Photo:
original_url=photo_dict["original_url"],
)
- def get_download_tasks(self) -> List[Task]:
+ async def urlretrieve(self, url, filename, session):
+ print(f"Retrieving {url}")
+ async with session.get(url) as resp:
+ if resp.status == 200:
+ with open(filename, 'wb') as picfile:
+ body = await resp.read()
+ print(f"Saving {filename}")
+ picfile.write(body)
+
+ def get_download_tasks(self, session) -> List[Task]:
makedirs("photos/originales", exist_ok=True)
makedirs("photos/filtrées", exist_ok=True)
return [
- create_task(self._download_original()),
- create_task(self._download_filtered()),
+ create_task(self._download_original(session)),
+ create_task(self._download_filtered(session)),
]
- async def _download_original(self) -> None:
- urlretrieve(self.original_url, f"photos/originales/{self.taken_at}")
+ async def _download_original(self, session) -> None:
+ await self.urlretrieve(self.original_url, f"photos/originales/{self.taken_at}", session)
- async def _download_filtered(self) -> None:
- urlretrieve(self.original_url, f"photos/filtrées/{self.taken_at}")
+ async def _download_filtered(self, session) -> None:
+ await self.urlretrieve(self.url, f"photos/filtrées/{self.taken_at}", session)
async def main() -> None:
@@ -72,9 +83,9 @@ async def main() -> None:
html_parser = CheezPageParser()
html_parser.feed(cheerz_page)
photos = (Photo.from_dict(d) for d in html_parser.photos_dict["photoData"])
-
- # Create and wait for all the download tasks
- await wait(chain.from_iterable(p.get_download_tasks() for p in photos))
+ async with aiohttp.ClientSession() as session:
+ # Create and wait for all the download tasks
+ await wait(chain.from_iterable(p.get_download_tasks(session) for p in photos))
if __name__ == "__main__":
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment