A Python implementation of the data structures used for @skyrising's mc-versions project

versions.py

from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from enum import Enum, auto
from typing import Optional, Union
import requests

class VersionType(Enum):
    release = auto()
    snapshot = auto()
    old_beta = auto()
    old_alpha = auto()
    alpha_server = auto()
    classic_server = auto()
    pending = auto()

    @classmethod
    def from_string(cls, string: str) -> VersionType:
        return {
            "release": cls.release,
            "snapshot": cls.snapshot,
            "old_beta": cls.old_beta,
            "old_alpha": cls.old_alpha,
            "alpha_server": cls.alpha_server,
            "classic_server": cls.classic_server,
            "pending": cls.pending,
        }[string]

class WorldFormat(Enum):
    classic = auto()
    indev = auto()
    alpha = auto()
    mcregion = auto()
    anvil = auto()

    @classmethod
    def from_string(cls, string: str) -> WorldFormat:
        return {
            "classic": cls.classic,
            "indev": cls.indev,
            "alpha": cls.alpha,
            "region": cls.mcregion,
            "anvil": cls.anvil,
        }[string]

@dataclass
class Download:
    sha1: str
    size: int
    url: str

    @classmethod
    def from_dict(cls, data: dict) -> Download:
        return cls(data["sha1"], data["size"], data["url"])

@dataclass
class ExtraDataManifest:
    asset_hash: Optional[str]
    asset_index: Optional[str]
    downloads_hash: Optional[str]
    downloads_id: int
    hash: str
    last_modified: Optional[datetime]
    time: Optional[datetime]
    type: VersionType
    url: str

    @classmethod
    def from_dict(cls, data: dict) -> ExtraDataManifest:
        return cls(
            data["assetHash"] if "assetHash" in data else None,
            data["assetIndex"] if "assetIndex" in data else None,
            data["downloadsHash"] if "downloadsHash" in data else None,
            data["downloadsId"],
            data["hash"],
            datetime.fromisoformat(data["lastModified"]) if "lastModified" in data else None,
            datetime.fromisoformat(data["time"]) if "time" in data else None,
            VersionType.from_string(data["type"]),
            data["url"],
        )

@dataclass
class Protocol:
    type: str
    version: int

    @classmethod
    def from_dict(cls, data: dict) -> Protocol:
        return cls(data["type"], data["version"])

@dataclass
class ExtraVersionInfo:
    is_client: bool
    downloads: dict[str, Download]
    id: str
    manifests: list[ExtraDataManifest]
    next_versions: list[str]
    normalized_version: str
    previous_versions: list[str]
    protocol: Optional[Protocol]
    release_target: Optional[str]
    release_time: datetime
    is_server: bool
    shared_mappings: bool
    world_format: Optional[WorldFormat]

    @classmethod
    def from_dict(cls, data: dict) -> ExtraVersionInfo:
        return cls(
            data["client"],
            {key: Download.from_dict(value) for key, value in data["downloads"].items()},
            data["id"],
            [ExtraDataManifest.from_dict(value) for value in data["manifests"]],
            data["next"],
            data["normalizedVersion"],
            data["previous"],
            Protocol.from_dict(data["protocol"]) if data.get("protocol") else None,  # the first instance of a null value in the json!
            data["releaseTarget"] if "releaseTarget" in data else None,
            datetime.fromisoformat(data["releaseTime"]),
            data["server"],
            data["sharedMappings"],
            WorldFormat.from_string(data["world"]["format"]) if "world" in data else None,
        )

@dataclass
class OSRule:
    name: Optional[str]
    version: Optional[str]
    arch: Optional[str]

    @classmethod
    def from_dict(cls, data: dict) -> OSRule:
        return cls(data.get("name"), data.get("version"), data.get("arch"))

@dataclass
class Rule:
    allow: bool
    features: Optional[dict[str, bool]]
    os: Optional[OSRule]

    @classmethod
    def from_dict(cls, data: dict) -> Rule:
        return cls(
            data["action"] == "allow",
            data.get("features"),
            OSRule.from_dict(data["os"]) if "os" in data else None,
        )

@dataclass
class JvmArgument:
    value: Union[str, list[str]]
    rules: list[Rule]

    @classmethod
    def from_dict(cls, data: dict) -> JvmArgument:
        return cls(data["value"], [Rule.from_dict(rule) for rule in data["rules"]] if "rules" in data else [])

@dataclass
class JvmArguments:
    game: list[Union[str, JvmArgument]]
    jvm: list[Union[str, JvmArgument]]

    @classmethod
    def from_dict(cls, data: dict) -> JvmArguments:
        game_data = data["game"]
        jvm_data = data["jvm"]
        for i, arg in enumerate(game_data):
            if isinstance(arg, dict):
                game_data[i] = JvmArgument.from_dict(arg)
        for i, arg in enumerate(jvm_data):
            if isinstance(arg, dict):
                jvm_data[i] = JvmArgument.from_dict(arg)
        return cls(game_data, jvm_data)

@dataclass
class AssetIndex:
    id: str
    sha1: str
    size: int
    total_size: int
    url: str

    @classmethod
    def from_dict(cls, data: dict) -> AssetIndex:
        return cls(data["id"], data["sha1"], data["size"], data["totalSize"], data["url"])

@dataclass
class JavaVersion:
    component: str
    major_version: int

    @classmethod
    def from_dict(cls, data: dict) -> JavaVersion:
        return cls(data["component"], data["majorVersion"])

# Stub to use in the Library class then actually define later
def native_lib(data: dict) -> Library: ...  # type: ignore
def maven_lib(data: dict) -> Library: ...  # type: ignore

@dataclass
class Library:
    path: str
    sha1: str
    size: int
    url: str
    name: str
    rules: list[Rule]

    @classmethod
    def from_dict(cls, data: dict) -> Library:
        if "natives" in data:
            return native_lib(data)
        elif "downloads" not in data:
            return maven_lib(data)

        return cls(
            data["downloads"]["artifact"]["path"],
            data["downloads"]["artifact"]["sha1"],
            data["downloads"]["artifact"]["size"],
            data["downloads"]["artifact"]["url"],
            data["name"],
            [Rule.from_dict(rule) for rule in data["rules"]] if "rules" in data else [],
        )

@dataclass
class NativeLibrary(Library):
    natives: dict[str, str]
    classifiers: Optional[dict[str, Download]]
    extract: Optional[dict[str, str]]

    @classmethod
    def from_dict(cls, data: dict) -> NativeLibrary:
        natives = data["natives"]
        classifiers = {k: Download.from_dict(v) for k, v in data["downloads"]["classifiers"].items()} if "downloads" in data and "classifiers" in data["downloads"] else None
        classifier = next(iter(classifiers.values())) if classifiers else None
        return cls(
            "",
            classifier.sha1 if classifier else "",
            classifier.size if classifier else 0,
            classifier.url if classifier else "",
            data["name"],
            [Rule.from_dict(rule) for rule in data["rules"]] if "rules" in data else [],
            natives,
            classifiers,
            data["extract"] if "extract" in data else None,
        )

@dataclass
class MavenLibrary(Library):
    @classmethod
    def from_dict(cls, data: dict) -> MavenLibrary:
        return cls(
            "",
            "",
            0,
            "",
            data["name"],
            [Rule.from_dict(rule) for rule in data["rules"]] if "rules" in data else [],
        )

def native_lib(data: dict) -> NativeLibrary: 
    return NativeLibrary.from_dict(data)

def maven_lib(data: dict) -> MavenLibrary:
    return MavenLibrary.from_dict(data)

class LoggingSide(Enum):
    client = auto()
    server = auto()

    @classmethod
    def from_string(cls, string: str) -> LoggingSide:
        return {
            "client": cls.client,
            "server": cls.server,
        }[string]

@dataclass
class Logging:
    argument: str
    file_id: str
    file_sha1: str
    file_size: int
    file_url: str
    type: str

    @classmethod
    def from_dict(cls, data: dict) -> Logging:
        return cls(
            data["argument"],
            data["file"]["id"],
            data["file"]["sha1"],
            data["file"]["size"],
            data["file"]["url"],
            data["type"],
        )

@dataclass
class Manifest:
    arguments: Optional[JvmArguments]
    asset_index: Optional[AssetIndex]
    assets: Optional[str]
    compliance_level: Optional[int]
    downloads: dict[str, Download]
    id: str
    java_version: Optional[JavaVersion]
    libraries: list[Library]
    logging: Optional[dict[LoggingSide, Logging]]
    main_class: Optional[str]
    minimum_launcher_version: Optional[int]
    release_time: datetime
    time: Optional[datetime]
    type: VersionType

    @classmethod
    def from_dict(cls, data: dict) -> Manifest:
        return cls(
            JvmArguments.from_dict(data["arguments"]) if "arguments" in data else None,
            AssetIndex.from_dict(data["assetIndex"]) if "assetIndex" in data else None,
            data["assets"] if "assets" in data else None,
            data["complianceLevel"] if "complianceLevel" in data else None,
            {key: Download.from_dict(value) for key, value in data["downloads"].items()},
            data["id"],
            JavaVersion.from_dict(data["javaVersion"]) if "javaVersion" in data else None,
            [Library.from_dict(value) for value in data["libraries"]],
            {LoggingSide.from_string(key): Logging.from_dict(value) for key, value in data["logging"].items()} if "logging" in data else None,
            data["mainClass"] if "mainClass" in data else None,
            data["minimumLauncherVersion"] if "minimumLauncherVersion" in data else None,
            datetime.fromisoformat(data["releaseTime"].replace("T24:", "T00:")),  # Fix for Classic c0.27_st
            datetime.fromisoformat(data["time"]) if "time" in data else None,
            VersionType.from_string(data["type"]),
        )

@dataclass
class Version:
    id: str
    omni_id: str
    type: VersionType
    url: str
    time: Optional[datetime]
    release_time: datetime
    details_url: str

    _manifest: Optional[Manifest] = field(init=False, repr=False, default=None)
    _details: Optional[ExtraVersionInfo] = field(init=False, repr=False, default=None)

    @property
    def manifest(self) -> Manifest:
        """Lazy load the manifest."""
        if self._manifest is None:
            if "%" in self.url:
                url = self.url.replace("%", "%25")  # funky
            else:
                url = self.url
            with requests.get(url) as response:
                self._manifest = Manifest.from_dict(response.json())

        return self._manifest

    @property
    def details(self) -> ExtraVersionInfo:
        """Lazy load extra version info."""
        if self._details is None:
            with requests.get(self.details_url) as response:
                self._details = ExtraVersionInfo.from_dict(response.json())

        return self._details
    
    @classmethod
    def from_dict(cls, data: dict) -> Version:
        return cls(
            data["id"],
            data["omniId"],
            VersionType.from_string(data["type"]),
            data["url"],
            datetime.fromisoformat(data["time"]) if "time" in data else None,
            datetime.fromisoformat(data["releaseTime"]),
            data["details"],
        )

@dataclass
class VersionManifest:
    latest: dict[str, str]
    versions: list[Version]

    _versions_by_id: dict[str, Version] = field(init=False, default_factory=dict)

    def __getitem__(self, name: str) -> Optional[Version]:
        if name in self._versions_by_id:
            return self._versions_by_id[name]

        for version in self.versions:
            if version.omni_id == name:
                self._versions_by_id[name] = version
                return version

        return None

    @classmethod
    def from_dict(cls, data: dict) -> VersionManifest:
        instance = cls(
            data["latest"],
            [Version.from_dict(version) for version in data["versions"]],
        )
        for version in instance.latest.values():
            instance[version]  # Add all the latest versions to the cache
        return instance

def get_version_manifest(root_url: str) -> VersionManifest:
    """Get the version manifest."""
    with requests.get(root_url + "version_manifest.json") as response:
        return VersionManifest.from_dict(response.json())

def test():
    from os import path
    manifest = get_version_manifest("https://skyrising.github.io/mc-versions/")
    if path.exists("last"):
        with open("last") as file:
            last_version = file.read()
        skip = True
    else:
        last_version = None
        skip = False
    try:
        for version in manifest.versions:
            if version.id == last_version:
                skip = False
                print(f'(Skipping {version.id} because it was the most recent version last time)')
            if skip:
                print(f'(Skipping {version.id} because it was already processed)')
                continue
            # Double-check that both the manifest and details can be loaded
            version.manifest
            version.details
            print(f'Checked {version.id} ({version.type})')
            last_version = version.id
    except Exception as e:
        with open("last", "w") as f:
            f.write(str(last_version))
        print(f'Picking up at {last_version} next time')
        raise e

if __name__ == "__main__":
    test()