kube_mirror.py

#!/usr/bin/env python3
"""Downloads the binaries of the three latest minor releases for kube{let,adm,ctl}, crictl and cni plugins"""

import re
import os
import tarfile
import requests

K8S_MAJOR_VERSION = 1
K8S_BINARIES = ['kubeadm', 'kubectl', 'kubelet']


def fetch_releases(repo):
    """Return json of all releases for repo"""
    print('Fetching releases for: ' + repo)
    response = requests.get('https://api.github.com/repos/' + repo + '/releases')
    return response.json()

def get_latest_major(json):
    """Return the latest major version"""
    latest_major = 0
    for release in json:
        major_version = int(re.sub(r'^v(\d+)\.\d+\.\d+(?:-.+)?', '\\1', release['tag_name']))
        if major_version > latest_major:
            latest_major = major_version
    return latest_major

def get_latest_minor(json, major, prefix=False):
    """Return the latest minor version for given major version"""
    latest_minor = 0
    for release in json:
        if prefix:
            is_stable = re.match(prefix, release['target_commitish'])
            if bool(is_stable):
                minor_version = int(re.sub(prefix, '\\1', release['target_commitish']))
                if minor_version > latest_minor:
                    latest_minor = minor_version
        else:
            is_major = re.match('^v' + str(major) + r'\.\d+\.\d+$', release['tag_name'])
            if bool(is_major):
                minor_version = int(re.sub('^v' + str(major) + r'\.(\d+)\.\d+', '\\1', release['tag_name']))
                if minor_version > latest_minor:
                    latest_minor = minor_version
    return latest_minor

def get_latest_patch(releases_json, major_version, minor_version):
    """Return the latest patch version for given minor version"""
    latest_patch = 0
    for release in releases_json:
        is_minor = re.match('^v' + str(major_version) + r'\.' + str(minor_version) + r'\.\d+$', release['tag_name'])
        if bool(is_minor):
            patch_version = int(re.sub('^v' + str(major_version) + r'\.' + str(minor_version) + r'\.(\d+)$', '\\1', release['tag_name']))
            if patch_version > latest_patch:
                latest_patch = patch_version
    return latest_patch

def download_file(path, file, url):
    """Download given file and extract if gzipped"""
    open(path + file, 'wb').write(requests.get(url).content)
    print(file + ' successfully downloaded')
    if file.endswith('.tar.gz') or file.endswith('.tgz'):
        tar = tarfile.open(path + file)
        tar.extractall(path=path)
        tar.close()
        os.remove(path + file)
        print(file + ' successfully extracted', end='\n\n')

def create_version_file(path, name, version):
    """Create hidden version file"""
    open(path + '.' + name + '-version', 'w').write(version)

def check_version_file(path, name, version):
    """Return whether hidden version file exists and is equal to given version"""
    try:
        version_file = open(path + '.' + name + '-version', 'r')
        downloaded_version = version_file.read()
        version_file.close()
        if downloaded_version == version:
            print(name + ' ' + version + ' is already up-to-date')
            return True
        return False
    except FileNotFoundError:
        print('')
        print(path + '.' + name + '-version' + ' not found')
        return False


K8S_RELEASES_JSON = fetch_releases('kubernetes/kubernetes')

K8S_LATEST_MINOR = get_latest_minor(K8S_RELEASES_JSON, K8S_MAJOR_VERSION, prefix=r'^release-1\.(\d+)$')

print('Latest stable version: v' + str(K8S_MAJOR_VERSION) + '.' + str(K8S_LATEST_MINOR))

K8S_MINOR_VERSION_LIST = [K8S_LATEST_MINOR, K8S_LATEST_MINOR-1, K8S_LATEST_MINOR-2]
print('Gathering the latest patches for the 3 latest minor versions: ' + str(K8S_MINOR_VERSION_LIST), end='\n\n')

FULL_VERSION_LIST = []
for minor in K8S_MINOR_VERSION_LIST:
    k8s_latest_patch = get_latest_patch(K8S_RELEASES_JSON, K8S_MAJOR_VERSION, minor)

    print('Latest v' + str(K8S_MAJOR_VERSION) + '.' + str(minor) + ' release: v' + str(K8S_MAJOR_VERSION) + '.' + str(minor) + '.' + str(k8s_latest_patch), end='\n\n')
    FULL_VERSION_LIST.insert(len(FULL_VERSION_LIST), 'v' + str(K8S_MAJOR_VERSION) + '.' + str(minor) + '.' + str(k8s_latest_patch))

print('Generating folder structure', end='\n\n')
for folder in K8S_MINOR_VERSION_LIST:
    minor_path = '1.' + str(folder) + '/'
    try:
        os.makedirs(minor_path, exist_ok=True)
    except OSError:
        print("Creation of the directory %s failed" % minor_path)

for k8s_version in FULL_VERSION_LIST:

    download_url = 'https://storage.googleapis.com/kubernetes-release/release/' + k8s_version + '/bin/linux/amd64/'
    minor_path = re.sub('^v(' + str(K8S_MAJOR_VERSION) + r'\.\d+)\.\d+$', '\\1', k8s_version) + '/'

    for binary in K8S_BINARIES:
        if check_version_file(minor_path, binary, k8s_version):
            continue

        print('Downloading ' + binary)
        download_file(minor_path, binary, download_url + binary)

        print('Setting permissions')
        os.chmod(minor_path + binary, 0o755)
        # create hidden version file
        create_version_file(minor_path, binary, k8s_version)
    print('')

CRI_TOOLS_JSON = fetch_releases('kubernetes-sigs/cri-tools')

CRI_TOOLS_VERSION_LIST = []
for minor in K8S_MINOR_VERSION_LIST:
    cri_latest_patch = get_latest_patch(CRI_TOOLS_JSON, K8S_MAJOR_VERSION, minor)

    print('Latest v' + str(K8S_MAJOR_VERSION) + '.' + str(minor) + ' release: v' + str(K8S_MAJOR_VERSION) + '.' + str(minor) + '.' + str(cri_latest_patch), end='\n\n')
    CRI_TOOLS_VERSION_LIST.insert(len(FULL_VERSION_LIST), 'v' + str(K8S_MAJOR_VERSION) + '.' + str(minor) + '.' + str(cri_latest_patch))

for cri_tools_version in CRI_TOOLS_VERSION_LIST:
    minor_path = re.sub(r'^v(' + str(K8S_MAJOR_VERSION) + r'\.\d+)\.\d+$', '\\1', cri_tools_version) + '/'
    for tag in CRI_TOOLS_JSON:
        if tag['tag_name'] == cri_tools_version:
            for asset in tag['assets']:
                is_linux_amd64 = re.match(r'^crictl-.+-linux-amd64\.tar\.gz$', asset['name'])
                if bool(is_linux_amd64):
                    if check_version_file(minor_path, 'crictl', cri_tools_version):
                        continue

                    print('Downloading crictl')
                    download_file(minor_path, 'crictl.tar.gz', asset['browser_download_url'])
                    create_version_file(minor_path, 'crictl', cri_tools_version)

print('')

CNI_PLUGINS_JSON = fetch_releases('containernetworking/plugins')

CNI_LATEST_MAJOR = get_latest_major(CNI_PLUGINS_JSON)
CNI_LATEST_MINOR = get_latest_minor(CNI_PLUGINS_JSON, CNI_LATEST_MAJOR)
CNI_LATEST_PATCH = get_latest_patch(CNI_PLUGINS_JSON, CNI_LATEST_MAJOR, CNI_LATEST_MINOR)

CNI_VERSION = 'v' + str(CNI_LATEST_MAJOR) + '.' + str(CNI_LATEST_MINOR) + '.' + str(CNI_LATEST_PATCH)

print('Latest stable version: ' + CNI_VERSION)

if not check_version_file('', 'cni', CNI_VERSION):
    download_file('cni', 'cni-plugins-linux-amd64-' + CNI_VERSION + '.tgz', 'https://github.com/containernetworking/plugins/releases/download/' + CNI_VERSION + '/' + 'cni-plugins-linux-amd64-' + CNI_VERSION + '.tgz')
    create_version_file('', 'cni', CNI_VERSION)