Attempt to download all the Etcher releases, to verify that the S3 links are correct

download_all_releases.py

#!/usr/bin/env python
from __future__ import print_function
import requests
import re
import os

github_project = 'resin-io/etcher'

releases_url = 'https://github.com/' + github_project + '/releases'
aws_url = 'https://resin-production-downloads.s3.amazonaws.com/etcher'

def download_file(url, filename):
    if os.path.exists(filename):
        print("%s already exists, skipping" % filename)
    else:
        print("Downloading %s" % url)
        r = requests.get(url, stream=True)
        with open(filename+'.tmp', 'wb') as f:
            for chunk in r.iter_content(chunk_size=10240):
                if chunk:
                    f.write(chunk)
	os.rename(filename+'.tmp', filename)

def download_releases_from(url):
    current_release = None
    for line in requests.get(url).text.split('\n'):
        m = re.search('<a href="/%s/releases/tag/(.*?)">(.*?)</a>' % github_project, line)
        if m:
            assert m.group(1) == m.group(2)
            release = m.group(1)
            assert release[0] == 'v'
            current_release = release[1:]
            print("Found release %s" % release)
            if not os.path.exists(release):
                os.mkdir(release)
            for filename in (release + '.tar.gz', release + '.zip'):
                download_file('https://github.com/' + github_project + '/archive/' + filename, os.path.join(release, filename))
        m = re.search('<li><a href="(%s/(.*?)/(.*?))">(.*?)</a></li>' % re.escape(aws_url), line)
        if m:
            assert current_release is not None
            file_url = m.group(1)
            assert m.group(2) == current_release
            filename = m.group(3)
            assert m.group(4) == filename
            #print("Release: %s Filename: %s" % (current_release, filename))
            download_file(file_url, os.path.join(release, filename))
        m = re.search('<a href="(%s\?after=(.*?))" rel="nofollow">Next</a>' % releases_url, line)
        if m:
            next_url = m.group(1)
            assert m.group(2) == release
            download_releases_from(next_url)

download_releases_from(releases_url)