BaNaNaS maintenance tools

blacklist.sh

#!/bin/sh
# blacklist.sh branch-name (content-type/unique-id)+

NAME="$1"
shift

git fetch upstream
git checkout master -b "blacklist_$NAME"
git rebase upstream/master

for l in $*
do
git rm -r "$l/*"
mkdir -p "$l"
echo "blacklisted: true" > "$l/global.yaml"
git add "$l/global.yaml"
done

convert.py

#!/usr/bin/env python3
import json
import os
import urllib3


def main():
    openttd = input("What is the OpenTTD username? ")
    github = input("What is the GitHub profile URL or username? ")

    if github.startswith("http"):
        github = github.split("/")[-1]

    pool = urllib3.PoolManager()
    response = pool.request("GET", "https://api.github.com/users/" + github, headers={'User-Agent': 'OpenTTD/Migration 1.0'})
    if response.status != 200:
        print("Couldn't find GitHub username " + github)
        return
    data = json.loads(response.data)
    github_id = str(data["id"])

    os.system("git fetch upstream")
    os.system("git checkout master -b user_" + openttd)
    os.system("git rebase upstream/master")
    os.system("sed -i 's/openttd: \"" + openttd + "\"/github: \"" + github_id + "\"/g' */*/authors.yaml")
    os.system("git add ai ai-library base-graphics base-music base-sounds game-script game-script-library heightmap newgrf scenario")
    os.system("git commit -m 'Change: migrate OpenTTD user " + openttd + " to GitHub user " + github + "'")
    os.system("git push")

    print("Follow git instructions to make a pull-request out of this")

if __name__ == "__main__":
    main()

data.py

import logging
import os
import yaml

from collections import OrderedDict
from enum import Enum


class ContentType(Enum):
    AI = "ai"
    AI_LIBRARY = "ai-library"
    BASE_GRAPHICS = "base-graphics"
    BASE_MUSIC = "base-music"
    BASE_SOUNDS = "base-sounds"
    GAME_SCRIPT = "game-script"
    GAME_SCRIPT_LIBRARY = "game-script-library"
    HEIGHTMAP = "heightmap"
    NEWGRF = "newgrf"
    SCENARIO = "scenario"


class key_string(str):
    pass


class date_string(str):
    pass


def string_presenter(dumper, data):
    if "\n" in data:
        return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
    return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"')


# Patch YAML to represent our data correctly
yaml.add_representer(OrderedDict, lambda dumper, data: dumper.represent_mapping("tag:yaml.org,2002:map", data.items()))
yaml.add_representer(key_string, lambda dumper, data: dumper.represent_scalar("tag:yaml.org,2002:str", data, style=""))
yaml.add_representer(date_string, lambda dumper, data: dumper.represent_scalar("tag:yaml.org,2002:timestamp", data))
yaml.add_representer(str, string_presenter)


class Index:
    def __init__(self, folder):
        self.folder = folder
        self.packages = []
        self.load_all()

    def _read_content_entry(self, content_type, folder_name, unique_id):
        folder_name = f"{folder_name}/{unique_id}"

        with open(f"{folder_name}/global.yaml") as f:
            package_data = yaml.safe_load(f.read())

        if package_data.get("blacklisted"):
            return None

        package_data["authors"] = []
        with open(f"{folder_name}/authors.yaml") as f:
            authors_data = yaml.safe_load(f.read())

            for author_data in authors_data.get("authors", []):
                package_data["authors"].append(author_data)

        package_data["versions"] = []
        for version in os.listdir(f"{folder_name}/versions"):
            with open(f"{folder_name}/versions/{version}") as f:
                version_data = yaml.safe_load(f.read())

                # YAML converts this in a datetime() for us, and marshmallow
                # expects a string. So output it as an ISO-8601 again.
                version_data["upload-date"] = version_data["upload-date"].isoformat()

                package_data["versions"].append(version_data)

        package_data["content-type"] = content_type.value
        package_data["unique-id"] = unique_id

        return package_data

    def load_all(self):
        for content_type in ContentType:
            folder_name = f"{self.folder}/{content_type.value}"

            if not os.path.isdir(folder_name):
                continue

            for unique_id in os.listdir(folder_name):
                try:
                    package = self._read_content_entry(content_type, folder_name, unique_id)
                except Exception:
                    print(f"Failed to load entry {folder_name}/{unique_id}. Skipping.")
                    continue

                self.packages.append(package)

packagelist.py

#!/usr/bin/env python3

import data
from collections import Counter

index = data.Index(".")

author_overview = []
type_histogram = []

for p in index.packages:
    if p is None:
        continue

    author_overview.append("\t".join((
        p["content-type"], p["unique-id"],
        p["name"],
        ", ".join("{}:{}:{}".format(a["display-name"], a.get("openttd", ""), a.get("github", "")) for a in p["authors"]),
        str(len(p["versions"])),
        min(v["upload-date"] for v in p["versions"]),
        max(v["upload-date"] for v in p["versions"])
    )))

    for v in p["versions"]:
        type_histogram.append((p["content-type"], v["upload-date"][0:7]))

author_overview.sort()
with open("author_overview.csv", "wt") as f:
    f.write("content-type\tunique-id\tname\tauthors\tnum-versions\tfirst\tlast\n")
    f.write("\n".join(author_overview))

header = sorted(v.value for v in data.ContentType.__members__.values())
histo = dict()
for h in type_histogram:
    k = header.index(h[0])
    d = histo.setdefault(h[1], [0] * len(header))
    d[k] = d[k] + 1

with open("type_histogram.csv", "wt") as f:
    f.write("date\t" + "\t".join(header) + "\n")
    for h in sorted(histo.items()):
        f.write("{}\t{}\n".format(h[0], "\t".join([str(c) for c in h[1]])))