Download Cloudfronted Bucket

download_cloudfronted_bucket.py

import requests
import xml.etree.ElementTree
import pprint
import re
import os
import threading

base_address = "https://xxxxxxxxxxxxxxxx.cloudfront.net/"

def createAndOpen(filename):
    try:
        os.makedirs(os.path.dirname(filename))
    except:
        pass
    return open(filename, 'w')

def download_file(key):
    content = requests.get(base_address + key).text.encode('utf-8')
    with createAndOpen(bucket_name + "/" + key) as f:
        f.write(content)
    print(key)
    return


r = requests.get(base_address)

xmlroot = xml.etree.ElementTree.fromstring(re.sub(' xmlns="[^"]+"', '', r.text, count=1))
bucket_name = xmlroot.find("Name").text

threads = []

for item in xmlroot.findall("Contents"):
    key = item.find("Key").text
    if not key.endswith("/"):
        t = threading.Thread(target=download_file, args=(key,))
        threads.append(t)
        t.start()