Get a list of files from https://cytomine.com/collection

_scrape_cytomine_collection.py

"""super quick script for getting the file urls for download"""
import sys
import requests
from bs4 import BeautifulSoup
from itertools import count

url = "https://cytomine.com/collection"

for page_idx in count():
    page = requests.get(url, params={"page": page_idx})
    soup = BeautifulSoup(page.content, "html.parser")

    section = soup.find(id="block-system-main")
    lis = list(section.find_all("li", class_="tabs-collection"))
    if not lis:
        break
    for li in lis:
        src = li.a.get("href")
        ipage = requests.get(src)
        isoup = BeautifulSoup(ipage.content, "html.parser")
        d = isoup.find("div", class_="cta-collection-btn-small-download")
        if not d:
            print("# skipping", src, file=sys.stderr)  # one file not available
        else:
            img_url = d.a.get("href")
            if img_url.startswith("http:"):
                img_url = f"https:{img_url[5:]}"  # one file has incorrect scheme
            print(img_url)

cytomine_collection_urls.txt

# python _scrape_cytomine_collection.py > cytomine_collection_urls.txt
# run on: 2022-04-08 19:30
https://data.cytomine.coop/open/openslide/aperio-svs/CMU-1-Small-Region.svs
https://data.cytomine.coop/open/uliege/earthworm-transv-posterior-to-clitellum-02.tiff
https://data.cytomine.coop/open/uliege/lombric-c-sagit-111.ndpi
https://data.cytomine.coop/open/uliege/lombric-c-sagit-107.ndpi
https://data.cytomine.coop/open/uliege/lombric-c-front-41.ndpi
https://data.cytomine.coop/open/uliege/lombric-c-front-30.ndpi
https://data.cytomine.coop/open/unamur/histology.be/HSM2001.tiff
https://data.cytomine.coop/open/unamur/histology.be/AP151.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0175.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0141.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0358.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0129.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0012.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0136.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0135.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0099.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0138.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0340.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0335.svs
https://data.cytomine.coop/open/uliege/Rectum_de_chat_HE.tif
https://data.cytomine.coop/open/unamur/histology.be/HSM0099.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0097.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0090.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0088.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0081.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0080.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0132.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0130.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0244.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0238.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0240.svs
https://data.cytomine.coop/open/bigtiff/HSM0170-intruder-20x20.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSV1001.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0097.svs
https://data.cytomine.coop/open/uclouvain/CT_feuille_dico_laurier-rose.scn
https://data.cytomine.coop/open/uclouvain/CT_feuille_mono_narcisse.scn
https://data.cytomine.coop/open/uclouvain/CT_racine_mono_iris.scn
https://data.cytomine.coop/open/uclouvain/CT_racine_dico_renoncule.scn
https://data.cytomine.coop/open/uclouvain/CT_tige_dico_rubus.scn
https://data.cytomine.coop/open/uclouvain/CT_tige_mono_mais.scn
https://data.cytomine.coop/open/unamur/histology.be/HSM0190.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0170.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0154.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0148.svs
https://data.cytomine.coop/open/onehealthphotography/ohp_ctenocephalidescanis_1.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSM0145.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0133.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSM0075.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSV0110.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSV0113-b-flip.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSM0087.tiff
https://data.cytomine.coop/open/unamur/histology.be/HSM0276.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0292.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0007.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0019.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0113-b.svs
https://data.cytomine.coop/open/unamur/histology.be/HSV0110.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0412.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0370.svs
https://data.cytomine.coop/open/unamur/histology.be/HSM0087.svs
https://data.cytomine.coop/open/openslide/aperio-svs/CMU-3.svs
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/OS-3.ndpi
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/OS-2.ndpi
https://data.cytomine.coop/open/openslide/hamamatsu-vms/CMU-3.zip
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/CMU-3.ndpi
https://data.cytomine.coop/open/openslide/aperio-svs/CMU-2.svs
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/OS-1.ndpi
https://data.cytomine.coop/open/openslide/ventana-bif/OS-2.bif
https://data.cytomine.coop/open/openslide/ventana-bif/OS-1.bif
https://data.cytomine.coop/open/openslide/aperio-svs/CMU-1.svs
https://data.cytomine.coop/open/openslide/hamamatsu-vms/CMU-2.zip
https://data.cytomine.coop/open/openslide/hamamatsu-vms/CMU-1.zip
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/CMU-2.ndpi
https://data.cytomine.coop/open/openslide/mirax-mrxs/CMU-2.zip
https://data.cytomine.coop/open/openslide/hamamatsu-ndpi/CMU-1.ndpi
https://data.cytomine.coop/open/openslide/generic-tiff/CMU-1.tiff
https://data.cytomine.coop/open/openslide/mirax-mrxs/CMU-1.zip