Python script to download missing files in a Yarn offline mirror

fixMissingOfflineMirrorFiles.py

# coding=utf-8
import sys
import re
import requests

if(sys.version_info.major < 3):
    print("This script must be run with Python 3.6 or higher!")
    exit(1)

import pathlib


rePackages = re.compile(r"resolved \"(?P<url>https.+\.com\/(?P<packageName>.+?)\/-\/(?P<tarball>.+-(?P<version>\d.+?)\.tgz)).+\"")


def getLockfileEntries(lockfilePath):
    text = open(lockfilePath).read()
    firstQuoteIndex = text.find('"')
    remainingText = text[firstQuoteIndex:]
    entries = remainingText.split("\n\n")
    return entries

def parseLockfileEntries(entries):
    packageEntries = [rePackages.search(e).groupdict() for e in entries]
    
    for package in packageEntries:
        pn = package["packageName"]
        tb = package["tarball"]
        url = package["url"]
        v = package["version"]
        
        actualTarball = tb
        if pn.startswith("@"):
            fixedName = pn.replace("/", "-")
            actualTarball = "{0}-{1}.tgz".format(fixedName, v)
        package["actualTarball"] = actualTarball

    return packageEntries 


def getMirrorFolderContents(mirrorPath):
    mirrorFiles = list(mirrorPath.iterdir())
    mirrorTarballs = set()
    for f in mirrorFiles:
        mirrorTarballs.add(f.name)

    return mirrorTarballs

def getLockfileTarballs(packageEntries):
    lockfileTarballs = set()
    for pe in packageEntries:
        lockfileTarballs.add(pe["actualTarball"])

    return lockfileTarballs

def calculateMissingFiles(mirrorTarballs, lockfileTarballs):
    missingFiles = lockfileTarballs - mirrorTarballs
    return missingFiles


def download(url, filename):
    with open(filename, 'wb') as f:
        response = requests.get(url, stream=True, verify=False)
        total = response.headers.get('content-length')

        if total is None:
            f.write(response.content)
        else:
            downloaded = 0
            total = int(total)
            for data in response.iter_content(chunk_size=max(int(total/1000), 1024*1024)):
                downloaded += len(data)
                f.write(data)
                done = int(50*downloaded/total)
                sys.stdout.write('\r[{}{}]'.format('█' * done, '.' * (50-done)))
                sys.stdout.flush()
    sys.stdout.write('\n')

def downloadMissingTarballs(missingFiles, packageEntries, mirrorPath):    
    for pe in packageEntries:
        actualTarball = pe["actualTarball"]
        url = pe["url"]
        if(actualTarball in missingFiles):
            print("Downloading: {0}".format(url))
            outputPath = mirrorPath / actualTarball
            download(url, str(outputPath))


def fixMissingOfflineMirrorFiles(projectFolder):
    projectPath = pathlib.Path(projectFolder).absolute()

    lockfilePath = projectPath / "yarn.lock"
    mirrorPath = projectPath / "offline-mirror"

    lockfileEntries = getLockfileEntries(lockfilePath)
    packageEntries = parseLockfileEntries(lockfileEntries)

    lockfileTarballs = getLockfileTarballs(packageEntries)
    mirrorTarballs = getMirrorFolderContents(mirrorPath)

    missingFiles = lockfileTarballs - mirrorTarballs

    print("Missing files: {0}".format(missingFiles))

    downloadMissingTarballs(missingFiles, packageEntries, mirrorPath)

if __name__ == "__main__":
    exe, projectFolder = sys.argv
    fixMissingOfflineMirrorFiles(projectFolder)