Old repo -> pkgbuild repo

failureswarns.json

{"run": 0, "fails": []}

repoconvert.py

import urllib.request
import json
import time
import os
import zipfile

t = time.time()
def GiveUp(package, reason):
    with open("failureswarns.json", "r+") as f:
        fail = json.load(f)
        if fail["run"] != t:
            fail = {"run": t, "fails": []}
        fail["fails"].append({"package": package, "reason": reason})
        f.seek(0)
        f.write(json.dumps(fail, indent=1))
        f.truncate()


domain = input("Repo domain: ")

plat = input("Repo platform (switch/wiiu): ")

summaryUrl = "http://" + domain + "/appstore/repo.summary"
repoUrl = "http://" + domain + "/appstore/repo.json"

repo = json.loads(urllib.request.urlopen(repoUrl).read().decode("utf-8"))['packages']

assetListDL = [[], []] # 0 = single binary, 1 = zips
usebk = input("Use release backup? (y/n): ")
if usebk.lower() == "y":
    with open(plat+"-releases.json", "r") as f:
        assetListDL = json.load(f)
else:
    print("Getting releases...")
    for i in range(0, len(repo)):
        if repo[i]["category"] == "theme":
            repo[i] = {}
            continue
        print("Processing: [" + str(i) + "] " + repo[i]["name"])
        urlS = repo[i]["url"].split("/")
        if len(urlS) < 5:
            print(" --> URL too short: " + repo[i]["url"])
            GiveUp(repo[i]["name"], "URL too short")
            continue

        if urlS[2] != "github.com":
            print(" --> URL not Github: " + repo[i]["url"])
            GiveUp(repo[i]["name"], "URL not Github")
            continue
        urlS = urlS[:5]
        urlS.append("releases/latest")
            
        urlS[2] = "api.github.com/repos"
        releaseAPI = '/'.join(urlS)
        resp = ""
        done = False
        fail = False
        while not done:
            try:
                resp = json.loads(urllib.request.urlopen(releaseAPI).read().decode("utf-8"))
                done = True
            except urllib.error.HTTPError as e:
                if e.code == 404:
                    print(" --> Not found :( Giving up")
                    GiveUp(repo[i]["name"], "Latest release API not found")
                    done = True
                    fail = True
                else:
                    print("====== " + e.reason + ", probably ran out of API quota, press enter to retry ======")
                    ent = input()
        if fail:
            continue

        assets = resp["assets"]
        if (len(assets) > 1):
            print(" --> Multiple assets, searching for NRO/ELF/RPX first, ZIP second")
            zCandidates = []
            binFound = False
            for asset in assets:
                if (".nro" in asset["name"] and plat == "switch") or ((".rpx" in asset["name"] or ".elf" in asset["name"]) and plat == "wiiu"):
                    assetListDL[0].append([repo[i], assets[0]["browser_download_url"]])
                    binFound = True
                    break
                if ".zip" in asset["name"]:
                    zCandidates.append([asset["name"], [repo[i], assets[0]["browser_download_url"]]])
            if len(zCandidates) > 1 and not binFound:
                print("     --> Multiple zip candidates, attempting platform match")
                success = False
                for c in zCandidates:
                    if plat in c[0]:
                        assetListDL[1].append(c[1])
                        success = True
                        break
                if not success:
                    print("         --> Failed plat match, gave up on package (first release name is: " + assets[0]["name"] + ")")
                    GiveUp(repo[i]["name"], "Failed to match zip name to a platform")
            elif len(zCandidates) > 0 and not binFound:
                assetListDL[1].append(zCandidates[0][1])
            elif not binFound:
                print("     --> Failed to find suitable release, gave up on package (first release name is: " + assets[0]["name"] + ")")
                GiveUp(repo[i]["name"], "Failed to find suitable release (no binary or zip)")
        else:
            try:
                if ".zip" in assets[0]["name"]:
                    assetListDL[1].append([repo[i], assets[0]["browser_download_url"]])
                else:
                    assetListDL[0].append([repo[i], assets[0]["browser_download_url"]])
            except IndexError:
                print(" --> Failed to find any release assets! Giving up.")
                GiveUp(repo[i]["name"], "No release assets found at all")
        time.sleep(.05)

    print("Finished getting releases! Backing up release file as '" + plat + "-releases.json'")
    with open(plat+"-releases.json", "w") as f:
        json.dump(assetListDL, f)



print("Downloading all zip releases...")
zipR = assetListDL[1]

for i in range(0, len(zipR)):
    print("Downloading: [" + str(i) + "/" + str(len(zipR)) + "] " + zipR[i][0]["name"])

    try:
        os.mkdir(zipR[i][0]["name"])
    except FileExistsError:
        pass

    while True:
        try:
            urllib.request.urlretrieve(zipR[i][1], zipR[i][0]["name"] + "/" + plat + ".zip")
        except urllib.error.HTTPError as e:
            print(" --> Error downloading: '" + e.reason + "', retrying...")
            time.sleep(1)
            continue
        except urllib.error.URLError as e:
            print(" --> Error downloading: '" + e.reason + "', retrying...")
            time.sleep(1)
            continue
        break

print("Downloading complete!")


print("Comparing manifest to zip file (single binary releases)...")

summary = json.loads(urllib.request.urlopen(summaryUrl).read().decode("utf-8"))
binR = assetListDL[0]

compSuccess = []

for i in range(0, len(binR)):
    try:
        os.mkdir(binR[i][0]["name"])
    except FileExistsError:
        pass
    print("Comparing: (Binary) [" + str(i) + "] " + binR[i][0]["name"])
    manifest = summary[binR[i][0]["name"]]
    if len(manifest) > 1:
        print(" --> More than one entry in manifest for single binary release! Giving up.")
        GiveUp(binR[i][0]["name"], "More than one entry in manifest when a single binary was provided")
    elif not binR[i][1].endswith(".zip"):
        entry = manifest[0].split(" ")
        asset = {"url": binR[i][1], "dest": entry[1], "type": ""}
        if entry[0] == "U:":
            asset["type"] = "update"
        elif entry[0] == "G:":
            asset["type"] = "get"
        elif entry[0] == "L:":
            asset["type"] = "local"
        elif entry[0] == "E:":
            asset["type"] = "extract"
        compSuccess.append([binR[i][0], asset])
    else:
        print(" --> Asset found on release was zip but expected binary! Giving up")
        GiveUp(binR[i][0]["name"], "Asset found on release was zip but expected binary")

print("Comparing manifest to zip file (zip releases)...")
zipR = assetListDL[1]
for i in range(0, len(zipR)):
    print("Comparing: (Zip) [" + str(i) + "] " + zipR[i][0]["name"])
    manifest = summary[zipR[i][0]["name"]]
    zf = ""
    try:
        zf = zipfile.ZipFile(zipR[i][0]["name"] + "/" + plat + ".zip", "r")
    except zipfile.BadZipFile:
        print(" --> Bad zip file! Giving up.")
        GiveUp(zipR[i][0]["name"], "Bad zip file")
        continue

    entries = []
    for uentry in manifest:
        entries.append(uentry.split(" "))
    
    zassets = []
    found = 0
    for f in zf.filelist:
            if f.filename.endswith("/"):
                continue
            asset = ""
            for entry in entries:
                if entry[1].lower() == f.filename.lower() or entry[1].lower() == plat+"/" + f.filename.lower():
                    found += 1
                    asset = {"path": f.filename, "dest": entry[1], "type": ""}
                    if entry[0] == "U:":
                        asset["type"] = "update"
                    elif entry[0] == "G:":
                        asset["type"] = "get"
                    elif entry[0] == "L:":
                        asset["type"] = "local"
                    elif entry[0] == "E:":
                        asset["type"] = "extract"
                    zassets.append(asset)
                    break
    
    if found != len(entries) and found != 0:
        print(" --> Some entries could not be automatically converted, may need manual checking")
        GiveUp(zipR[i][0]["name"], "[NOT FATAL] Some manifest entries could not be automatically converted, check manually")
    if found == 0:
        print(" --> No entries could be converted! Giving up.")
        GiveUp(zipR[i][0]["name"], "No manifest entries could be converted")
        continue

    zassetentry = {"url": zipR[i][1], "type": "zip", "zip": zassets}
    compSuccess.append([zipR[i][0], zassetentry])

print("Manifest compare complete")


print("Converting package meta json to pkgbuild")

for pkg in compSuccess:
    print("Converting: " + pkg[0]["name"])
    infoj = json.loads(urllib.request.urlopen("http://" + domain + "/appstore/packages/" + pkg[0]["name"] + "/info.json").read().decode("utf-8"))
    pkginfo = {"package": pkg[0]["name"], "info": {"title": infoj["title"], "author": infoj["author"], "category": infoj["category"], "version": infoj["version"], "url": infoj["url"], "license": infoj["license"], "description": infoj["description"], "details": infoj["details"]}, "changes": "", "assets": []}
    pkginfo["assets"] = [pkg[1]]
    try:
        pkginfo["changes"] = infoj["changelog"]
    except:
        pass
    try:
        urllib.request.urlretrieve("http://" + domain + "/appstore/packages/" + pkg[0]["name"] + "/icon.png", pkg[0]["name"] + "/icon.png")
        pkginfo["assets"].append({"url": "icon.png", "type": "icon"})
    except urllib.error.HTTPError as e:
        if e.code != 404:
            raise e
    try:
        urllib.request.urlretrieve("http://" + domain + "/appstore/packages/" + pkg[0]["name"] + "/screen.png", pkg[0]["name"] + "/screen.png")
        pkginfo["assets"].append({"url": "screen.png", "type": "screenshot"})
    except urllib.error.HTTPError as e:
        if e.code != 404:
            raise e
    
    with open(pkg[0]["name"] + "/pkgbuild." + plat + ".json", "w") as f:
        json.dump(pkginfo, f, indent=1)

print("Completed whole repo conversion!")

print("Cleaning up...")

for z in zipR:
    os.remove(z[0]["name"] + "/" + plat + ".zip")

folders = list(os.walk("./"))[1:]

for folder in folders:
    if not folder[2]:
        os.rmdir(folder[0])