Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Python script to download missing files in a Yarn offline mirror
# coding=utf-8
import sys
import re
import requests
if(sys.version_info.major < 3):
print("This script must be run with Python 3.6 or higher!")
exit(1)
import pathlib
rePackages = re.compile(r"resolved \"(?P<url>https.+\.com\/(?P<packageName>.+?)\/-\/(?P<tarball>.+-(?P<version>\d.+?)\.tgz)).+\"")
def getLockfileEntries(lockfilePath):
text = open(lockfilePath).read()
firstQuoteIndex = text.find('"')
remainingText = text[firstQuoteIndex:]
entries = remainingText.split("\n\n")
return entries
def parseLockfileEntries(entries):
packageEntries = [rePackages.search(e).groupdict() for e in entries]
for package in packageEntries:
pn = package["packageName"]
tb = package["tarball"]
url = package["url"]
v = package["version"]
actualTarball = tb
if pn.startswith("@"):
fixedName = pn.replace("/", "-")
actualTarball = "{0}-{1}.tgz".format(fixedName, v)
package["actualTarball"] = actualTarball
return packageEntries
def getMirrorFolderContents(mirrorPath):
mirrorFiles = list(mirrorPath.iterdir())
mirrorTarballs = set()
for f in mirrorFiles:
mirrorTarballs.add(f.name)
return mirrorTarballs
def getLockfileTarballs(packageEntries):
lockfileTarballs = set()
for pe in packageEntries:
lockfileTarballs.add(pe["actualTarball"])
return lockfileTarballs
def calculateMissingFiles(mirrorTarballs, lockfileTarballs):
missingFiles = lockfileTarballs - mirrorTarballs
return missingFiles
def download(url, filename):
with open(filename, 'wb') as f:
response = requests.get(url, stream=True, verify=False)
total = response.headers.get('content-length')
if total is None:
f.write(response.content)
else:
downloaded = 0
total = int(total)
for data in response.iter_content(chunk_size=max(int(total/1000), 1024*1024)):
downloaded += len(data)
f.write(data)
done = int(50*downloaded/total)
sys.stdout.write('\r[{}{}]'.format('█' * done, '.' * (50-done)))
sys.stdout.flush()
sys.stdout.write('\n')
def downloadMissingTarballs(missingFiles, packageEntries, mirrorPath):
for pe in packageEntries:
actualTarball = pe["actualTarball"]
url = pe["url"]
if(actualTarball in missingFiles):
print("Downloading: {0}".format(url))
outputPath = mirrorPath / actualTarball
download(url, str(outputPath))
def fixMissingOfflineMirrorFiles(projectFolder):
projectPath = pathlib.Path(projectFolder).absolute()
lockfilePath = projectPath / "yarn.lock"
mirrorPath = projectPath / "offline-mirror"
lockfileEntries = getLockfileEntries(lockfilePath)
packageEntries = parseLockfileEntries(lockfileEntries)
lockfileTarballs = getLockfileTarballs(packageEntries)
mirrorTarballs = getMirrorFolderContents(mirrorPath)
missingFiles = lockfileTarballs - mirrorTarballs
print("Missing files: {0}".format(missingFiles))
downloadMissingTarballs(missingFiles, packageEntries, mirrorPath)
if __name__ == "__main__":
exe, projectFolder = sys.argv
fixMissingOfflineMirrorFiles(projectFolder)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment