Skip to content

Instantly share code, notes, and snippets.

@ap--
Created April 8, 2022 17:38
Show Gist options
  • Save ap--/4143a86a73b83f717555cb88aa2dd96e to your computer and use it in GitHub Desktop.
Save ap--/4143a86a73b83f717555cb88aa2dd96e to your computer and use it in GitHub Desktop.
Get a list of files from https://cytomine.com/collection
"""super quick script for getting the file urls for download"""
import sys
import requests
from bs4 import BeautifulSoup
from itertools import count
url = "https://cytomine.com/collection"
for page_idx in count():
page = requests.get(url, params={"page": page_idx})
soup = BeautifulSoup(page.content, "html.parser")
section = soup.find(id="block-system-main")
lis = list(section.find_all("li", class_="tabs-collection"))
if not lis:
break
for li in lis:
src = li.a.get("href")
ipage = requests.get(src)
isoup = BeautifulSoup(ipage.content, "html.parser")
d = isoup.find("div", class_="cta-collection-btn-small-download")
if not d:
print("# skipping", src, file=sys.stderr) # one file not available
else:
img_url = d.a.get("href")
if img_url.startswith("http:"):
img_url = f"https:{img_url[5:]}" # one file has incorrect scheme
print(img_url)
# python _scrape_cytomine_collection.py > cytomine_collection_urls.txt
# run on: 2022-04-08 19:30
https://data.cytomine.coop/open/openslide/aperio-svs/CMU-1-Small-Region.svs
https://data.cytomine.coop/open/uliege/earthworm-transv-posterior-to-clitellum-02.tiff
https://data.cytomine.coop/open/uliege/lombric-c-sagit-111.ndpi
https://data.cytomine.coop/open/uliege/lombric-c-sagit-107.ndpi
https://data.cytomine.coop/open/uliege/lombric-c-front-41.ndpi
https://data.cytomine.coop/open/uliege/lombric-c-front-30.ndpi
https://data.cytomine.coop/open/unamur/histology.be/HSM2001.tiff
https://data.cytomine.coop/open/unamur/histology.be/AP151.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0175.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0141.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0358.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0129.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0012.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0136.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0135.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0099.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0138.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0340.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0335.svs
https://data.cytomine.coop/open/uliege/Rectum_de_chat_HE.tif
https://data.cytomine.coop/open/unamur/histology.be/HSM0099.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0097.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0090.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0088.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0081.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0080.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0132.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0130.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0244.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0238.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0240.svs
https://data.cytomine.coop/open/bigtiff/HSM0170-intruder-20x20.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSV1001.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0097.svs
https://data.cytomine.coop/open/uclouvain/CT_feuille_dico_laurier-rose.scn
https://data.cytomine.coop/open/uclouvain/CT_feuille_mono_narcisse.scn
https://data.cytomine.coop/open/uclouvain/CT_racine_mono_iris.scn
https://data.cytomine.coop/open/uclouvain/CT_racine_dico_renoncule.scn
https://data.cytomine.coop/open/uclouvain/CT_tige_dico_rubus.scn
https://data.cytomine.coop/open/uclouvain/CT_tige_mono_mais.scn
https://data.cytomine.coop/open/unamur/histology.be/HSM0190.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0170.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0154.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0148.svs
https://data.cytomine.coop/open/onehealthphotography/ohp_ctenocephalidescanis_1.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSM0145.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0133.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSM0075.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSV0110.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSV0113-b-flip.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSM0087.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSM0276.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0292.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0007.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0019.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0113-b.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0110.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0412.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0370.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0087.svs
https://data.cytomine.coop/open/openslide/aperio-svs/CMU-3.svs
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/OS-3.ndpi
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/OS-2.ndpi
https://data.cytomine.coop/open/openslide/hamamatsu-vms/CMU-3.zip
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/CMU-3.ndpi
https://data.cytomine.coop/open/openslide/aperio-svs/CMU-2.svs
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/OS-1.ndpi
https://data.cytomine.coop/open/openslide/ventana-bif/OS-2.bif
https://data.cytomine.coop/open/openslide/ventana-bif/OS-1.bif
https://data.cytomine.coop/open/openslide/aperio-svs/CMU-1.svs
https://data.cytomine.coop/open/openslide/hamamatsu-vms/CMU-2.zip
https://data.cytomine.coop/open/openslide/hamamatsu-vms/CMU-1.zip
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/CMU-2.ndpi
https://data.cytomine.coop/open/openslide/mirax-mrxs/CMU-2.zip
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/CMU-1.ndpi
https://data.cytomine.coop/open/openslide/generic-tiff/CMU-1.tiff
https://data.cytomine.coop/open/openslide/mirax-mrxs/CMU-1.zip
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment