CIS_Downloader.py

import requests
from pathlib import Path

base_url = 'https://downloads.cisecurity.org'
download_dir = Path.cwd() / 'CIS'

def download():
    resp = requests.get(base_url + '/technology')
    if resp.status_code != 200:
        return

    categories = resp.json()
    for category in categories:
        # Create category directory
        cat_dir = download_dir / category        
        cat_dir.mkdir(parents=True, exist_ok=True)

        # Foreach technology in category
        for technology in categories[category]:
            tech_dir = cat_dir / technology.get('title')
            tech_dir.mkdir(exist_ok=True)
            resp = requests.get(base_url + '/technology/%s/benchmarks/latest' % technology.get('id'))
            benchmarks = resp.json()

            # Few endpoints does not return list but dict.
            # With this condition we translate dict to list object
            if isinstance(benchmarks, dict):
                benchmarks = benchmarks.values()
            
            # Foreach benchmark we download every documents
            for benchmark in benchmarks:
                for document in benchmark.get('documents'):
                    cookies = {
                        'documentId': str(document.get('id'))
                    }
                    url = 'https://downloads.cisecurity.org/download?u=1603942000'
                    resp = requests.get(url, cookies=cookies)
                    filename = document.get('filename')
                    file_path = tech_dir / filename
                    file_path.write_bytes(resp.content)


if __name__ == '__main__':
    download()