Stormgate replay renamer - See https://github.com/acarapetis/shroudstone for newest version

rename_replays.py

#!/usr/bin/env python3
# This script works, but a nicer version is now available at https://github.com/acarapetis/shroudstone

from pathlib import Path
from typing import NamedTuple
from urllib.request import urlopen
from datetime import datetime, timezone, timedelta
from time import sleep
import json
import pandas as pd
import re

replay_dir = Path(
    # Set this to the correct path for your OS - you'll need to update your username and also the UUID at the end.
    # Here I'm using WSL, thus the /mnt/c. If you're running python directly on windows it'd start with C:\ or similar.
    r"/mnt/c/Users/Anthony/AppData/Local/Stormgate/Saved/Replays/c6b4eb4e-4994-4e96-b098-3e1953a02033"
)
my_player_id = (
    # Set this to your player ID.
    # To find this, find your player page by searching on https://stormgateworld.com/leaderboards/ranked_1v1
    # and then extract the six-character gibberish string from the URL.
    "R28aVH"
)
TOLERANCE = timedelta(seconds=90)


def main():
    unrenamed_replays = [
        x
        for x in map(ReplayFile.from_path, replay_dir.glob("CL*.SGReplay"))
        if x is not None
    ]
    if not unrenamed_replays:
        print(
            "No replays found to rename! "
            f"If you weren't expecting this, check your replay_dir '{replay_dir}' is correct."
        )
        return

    earliest_time = min(x.time for x in unrenamed_replays)
    matches = player_matches_since(my_player_id, earliest_time)

    for r in unrenamed_replays:
        match = find_match(matches, r)
        if match is not None:
            try:
                rename_replay(r, match)
            except Exception as e:
                print(f"Unexpected error handling {r.path}: {e}")
        else:
            print(f"No match found for {r.path.name} in stormgateworld match history.")


def player_matches_since(player_id: str, time: datetime):
    page = 1
    matches = player_matches(player_id, page=page)
    if matches is None:
        raise Exception(
            "Could not find any matches for {player_id=} - check this is correct!"
        )
    while matches.created_at.min() > time:
        sleep(1)  # Be nice to the API servers! You can wait a few seconds!
        print(f"Got matches back to {matches.created_at.min()}, going further.")
        page += 1
        next_page = player_matches(player_id, page=page)
        if next_page is None:
            print(
                "Warning: Could only fetch back to {matches.created_at.min()} "
                "from stormgateworld but you have unrenamed replays from before this.",
            )
            return matches.reset_index(drop=True)
        matches = pd.concat([matches, next_page])
    return matches.reset_index(drop=True)


def player_matches(player_id: str, page: int):
    url = f"https://api.stormgateworld.com/v0/players/{player_id}/matches?page={page}"
    print(f"Fetching match history page {page}...")
    with urlopen(url) as f:
        data = json.load(f)["matches"]
    if not data:
        return None
    data = [flatten_match(player_id, x) for x in data]
    matches = pd.json_normalize([x for x in data if x is not None])
    matches["ended_at"] = pd.to_datetime(matches["ended_at"])
    matches["created_at"] = pd.to_datetime(matches["created_at"])
    return matches


def flatten_match(player_id: str, match: dict):
    try:
        match["us"] = next(
            p for p in match["players"] if p["player"]["player_id"] == player_id
        )
        match["them"] = next(
            p for p in match["players"] if p["player"]["player_id"] != player_id
        )
    except Exception:
        return None
    del match["players"]
    return match


class ReplayFile(NamedTuple):
    path: Path
    time: datetime

    @staticmethod
    def from_path(path: Path):
        m = re.search(r"(\d\d\d\d)\.(\d\d)\.(\d\d)-(\d\d).(\d\d)", path.name)
        if not m:
            return None
        time = (
            datetime(*(int(x) for x in m.groups()))  # type: ignore
            .astimezone()
            .astimezone(timezone.utc)
            .replace(tzinfo=None)
        )
        return ReplayFile(path, time)


def find_match(matches: pd.DataFrame, replay: ReplayFile):
    delta = (matches.created_at - replay.time).abs()
    idxmin = delta.idxmin()
    if delta[idxmin] < TOLERANCE:
        return matches.iloc[idxmin]


def rename_replay(replay: ReplayFile, match):
    us = match["us.player.nickname"]
    them = match["them.player.nickname"]

    try:
        r1 = match["us.race"][0].upper()
    except Exception:
        r1 = "?"

    try:
        r2 = match["them.race"][0].upper()
    except Exception:
        r2 = "?"

    try:
        result = match["us.result"][0].upper()
    except Exception:
        result = "?"

    try:
        minutes, seconds = divmod(int(match["duration"]), 60)
        duration = f"{minutes:02d}m{seconds:02d}s"
        time = match["created_at"].strftime("%Y-%m-%d %H.%M")
    except Exception:
        duration = "??m??s"
        time = replay.time.strftime("%Y-%m-%d %H.%M")

    try:
        newname = f"{time} {result} {duration} {us} {r1}v{r2} {them}.SGReplay"
        target = replay.path.parent / newname
        do_rename(replay.path, target)
    except Exception as e:
        print(f"Unexpected error renaming {replay.path}: {e}")


def do_rename(source: Path, target: Path):
    if target.exists():
        print(f"Not renaming {source}! {target.name} already exists!")
        return

    print(f"Renaming {source} => {target.name}.")
    try:
        source.rename(target)
    except Exception as e:
        # In case the error was due to weird characters in a player name:
        new_name = sanitize_filename(target.name)
        if new_name != target.name:
            print(
                f"Error renaming {source} => {target.name}, retrying with sanitized filename."
            )
            do_rename(source, target.parent / new_name)
        else:
            print(f"Error renaming {source} => {target.name}: {e}")


# These characters are forbidden in filenames on Linux or Windows:
bad_chars = re.compile(r'[<>:"/\\|?*\0]')
def sanitize_filename(filename: str) -> str:
    """Remove bad characters from a filename"""
    return bad_chars.sub("", filename)


if __name__ == "__main__":
    main()

I hoped that it meant that replays are parsable : /

I hoped that it meant that replays are parsable : /

Haven't had much luck with that yet - they're protobuf, but the schema is not publically available and I had trouble getting much meaningful out of them with protoscope.

Silly question but you unzipped them first, right?

@Fafa87 Yeah, here's the command I was trying: tail -c +17 CL45737-2024.02.14-21.18.SGReplay | zcat | ~/go/bin/protoscope | less

For some replays this was at least extracting the map and player names:

3: 7.974551e-24i32  # 0x191a4010i32
1: {
  3: {
    2: {"BrokenCrown"}
    3: 1566072066
    6: 1
  }
}
5:EGROUP
2: 64
3: {
  1: {
    37: {
      1: {
        1: 6518020192259623528
        2: -5630106735107133817
      }
      2: 1
      3: {"Sturgeon"}
    }
  }
}
2: 16
8: 26
1:EGROUP
1: {
  15: {
    3: 1
    4: 2952722564
  }
}
5: 16
8: 26
4:EGROUP
1: {
  37: {
    1: {
      1: -4128416236917469546
      2: -5721755048076107725
    }
    2: 2
    3: {"Pox"}
  }
}
2: {`401a0e0a0c7a0a18022084f1fbff0a28`}
`010910401a050a038a01000b08f00510011a040a0222000b08fd0510021a040a0222000b08bb0710`
# rest of output is similar unstructured binary

But for other replays it would cut to the binary earlier, not finding the map and player names. From inspecting the files I can see those strings are in there, but protoscope just isn't seeing the structure right. I'm not well-versed in protobuf so don't really know what the issue could be here - let me know if you make any progress!

Heya, tried the script right now but <5% of my replays got renamed (only 19th-22th of February nothing prior). Does it not work for prior matches? Or what is the reason for this?

While Debugging its clear that the find_match function simply does not find a match because they dont appear in the "matches" dataframe

Edit: The matches API query is paged, can just iterate through all of the pages

Edit2: Might be important to rename some opponents if they have special character in their name that are not allowed in your OS

@WinckelData: Thanks for your comment!
When I wrote the script, only the first page of player matches was available. You're right, they've now added a page parameter to that endpoint so we should be able to go back as far as we need. I'll post an updated version soon :)

I haven't had any issues with filenames yet, but you're right that it could be a problem.

@acarapetis awesome work!

Still getting some errors for the username "And It went like | Moon" due to the vertical bar.

your sanitize_filename function just looks at the start of the string and deletes just the first letter.

FYI, the forbidden printable ASCII characters for filenames are:

Linux/Unix:

• / (forward slash)

Windows:

• < (less than)
• > (greater than)
• : (colon - sometimes works, but is actually NTFS Alternate Data Streams)
• " (double quote)
• / (forward slash)
• \ (backslash)
• | (vertical bar or pipe)
• ? (question mark)
• * (asterisk)

May i suggest something like

def sanitize_filename(filename: str) -> str:
    # Forbidden characters in filename
    pattern = r'[<>:"/\\|?*]'
    # Replace the characters with an empty string
    sanitized_filename = re.sub(pattern, "", filename)
    return sanitized_filename

@WinckelData Had a typo in the regex, fixed it in revision 3 but looks like you got in quick :)

Your approach is better though - should give more consistent results and I'm pretty sure we don't need to worry about any other platforms. I'll publish a new revision.

@Fafa87 worked out the replay files, there's an extra layer on top of the protobuf - after unzipping, what you have is a sequence of length-prefixed protobuf messages. (e.g. if the first byte is 0x30, the next 48 bytes are a protobuf message, with the byte following being the next length marker). Looks like we could get map name and player nicknames pretty reliably with that, anything more will be a struggle though.

Map names would be a great addition to the renaming! 👍

I'm now getting map names out of the replays successfully, as well as double-checking the player nicknames. It's grown a bit big to be a gist now, so I've published it as a python package - https://github.com/acarapetis/shroudstone

Awesome Work, Haven't tried it yet only went a bit over the code on my phonr but I would highly appreciate a way to customize the renaming schema If that is not already implemented

Read my mind, working on custom format strings already :)

Great work here. I tried this out on linux+proton and it just worked by doing the pip install and just running it.

I did get some errors of games not found but I think its because the replays were against ai so they didn't have a game logged on stormgateworld.

output:

                    ERROR    No match found for CL45737-2024.02.18-20.00.SGReplay in stormgateworld match history.                                              renamer.py:86
                    ERROR    No match found for CL45737-2024.02.18-20.03.SGReplay in stormgateworld match history.                                              renamer.py:86