-
-
Save acarapetis/b5e9cf199ea2a468ace2f756ac287268 to your computer and use it in GitHub Desktop.
#!/usr/bin/env python3 | |
# This script works, but a nicer version is now available at https://github.com/acarapetis/shroudstone | |
from pathlib import Path | |
from typing import NamedTuple | |
from urllib.request import urlopen | |
from datetime import datetime, timezone, timedelta | |
from time import sleep | |
import json | |
import pandas as pd | |
import re | |
replay_dir = Path( | |
# Set this to the correct path for your OS - you'll need to update your username and also the UUID at the end. | |
# Here I'm using WSL, thus the /mnt/c. If you're running python directly on windows it'd start with C:\ or similar. | |
r"/mnt/c/Users/Anthony/AppData/Local/Stormgate/Saved/Replays/c6b4eb4e-4994-4e96-b098-3e1953a02033" | |
) | |
my_player_id = ( | |
# Set this to your player ID. | |
# To find this, find your player page by searching on https://stormgateworld.com/leaderboards/ranked_1v1 | |
# and then extract the six-character gibberish string from the URL. | |
"R28aVH" | |
) | |
TOLERANCE = timedelta(seconds=90) | |
def main(): | |
unrenamed_replays = [ | |
x | |
for x in map(ReplayFile.from_path, replay_dir.glob("CL*.SGReplay")) | |
if x is not None | |
] | |
if not unrenamed_replays: | |
print( | |
"No replays found to rename! " | |
f"If you weren't expecting this, check your replay_dir '{replay_dir}' is correct." | |
) | |
return | |
earliest_time = min(x.time for x in unrenamed_replays) | |
matches = player_matches_since(my_player_id, earliest_time) | |
for r in unrenamed_replays: | |
match = find_match(matches, r) | |
if match is not None: | |
try: | |
rename_replay(r, match) | |
except Exception as e: | |
print(f"Unexpected error handling {r.path}: {e}") | |
else: | |
print(f"No match found for {r.path.name} in stormgateworld match history.") | |
def player_matches_since(player_id: str, time: datetime): | |
page = 1 | |
matches = player_matches(player_id, page=page) | |
if matches is None: | |
raise Exception( | |
"Could not find any matches for {player_id=} - check this is correct!" | |
) | |
while matches.created_at.min() > time: | |
sleep(1) # Be nice to the API servers! You can wait a few seconds! | |
print(f"Got matches back to {matches.created_at.min()}, going further.") | |
page += 1 | |
next_page = player_matches(player_id, page=page) | |
if next_page is None: | |
print( | |
"Warning: Could only fetch back to {matches.created_at.min()} " | |
"from stormgateworld but you have unrenamed replays from before this.", | |
) | |
return matches.reset_index(drop=True) | |
matches = pd.concat([matches, next_page]) | |
return matches.reset_index(drop=True) | |
def player_matches(player_id: str, page: int): | |
url = f"https://api.stormgateworld.com/v0/players/{player_id}/matches?page={page}" | |
print(f"Fetching match history page {page}...") | |
with urlopen(url) as f: | |
data = json.load(f)["matches"] | |
if not data: | |
return None | |
data = [flatten_match(player_id, x) for x in data] | |
matches = pd.json_normalize([x for x in data if x is not None]) | |
matches["ended_at"] = pd.to_datetime(matches["ended_at"]) | |
matches["created_at"] = pd.to_datetime(matches["created_at"]) | |
return matches | |
def flatten_match(player_id: str, match: dict): | |
try: | |
match["us"] = next( | |
p for p in match["players"] if p["player"]["player_id"] == player_id | |
) | |
match["them"] = next( | |
p for p in match["players"] if p["player"]["player_id"] != player_id | |
) | |
except Exception: | |
return None | |
del match["players"] | |
return match | |
class ReplayFile(NamedTuple): | |
path: Path | |
time: datetime | |
@staticmethod | |
def from_path(path: Path): | |
m = re.search(r"(\d\d\d\d)\.(\d\d)\.(\d\d)-(\d\d).(\d\d)", path.name) | |
if not m: | |
return None | |
time = ( | |
datetime(*(int(x) for x in m.groups())) # type: ignore | |
.astimezone() | |
.astimezone(timezone.utc) | |
.replace(tzinfo=None) | |
) | |
return ReplayFile(path, time) | |
def find_match(matches: pd.DataFrame, replay: ReplayFile): | |
delta = (matches.created_at - replay.time).abs() | |
idxmin = delta.idxmin() | |
if delta[idxmin] < TOLERANCE: | |
return matches.iloc[idxmin] | |
def rename_replay(replay: ReplayFile, match): | |
us = match["us.player.nickname"] | |
them = match["them.player.nickname"] | |
try: | |
r1 = match["us.race"][0].upper() | |
except Exception: | |
r1 = "?" | |
try: | |
r2 = match["them.race"][0].upper() | |
except Exception: | |
r2 = "?" | |
try: | |
result = match["us.result"][0].upper() | |
except Exception: | |
result = "?" | |
try: | |
minutes, seconds = divmod(int(match["duration"]), 60) | |
duration = f"{minutes:02d}m{seconds:02d}s" | |
time = match["created_at"].strftime("%Y-%m-%d %H.%M") | |
except Exception: | |
duration = "??m??s" | |
time = replay.time.strftime("%Y-%m-%d %H.%M") | |
try: | |
newname = f"{time} {result} {duration} {us} {r1}v{r2} {them}.SGReplay" | |
target = replay.path.parent / newname | |
do_rename(replay.path, target) | |
except Exception as e: | |
print(f"Unexpected error renaming {replay.path}: {e}") | |
def do_rename(source: Path, target: Path): | |
if target.exists(): | |
print(f"Not renaming {source}! {target.name} already exists!") | |
return | |
print(f"Renaming {source} => {target.name}.") | |
try: | |
source.rename(target) | |
except Exception as e: | |
# In case the error was due to weird characters in a player name: | |
new_name = sanitize_filename(target.name) | |
if new_name != target.name: | |
print( | |
f"Error renaming {source} => {target.name}, retrying with sanitized filename." | |
) | |
do_rename(source, target.parent / new_name) | |
else: | |
print(f"Error renaming {source} => {target.name}: {e}") | |
# These characters are forbidden in filenames on Linux or Windows: | |
bad_chars = re.compile(r'[<>:"/\\|?*\0]') | |
def sanitize_filename(filename: str) -> str: | |
"""Remove bad characters from a filename""" | |
return bad_chars.sub("", filename) | |
if __name__ == "__main__": | |
main() |
@Fafa87 Yeah, here's the command I was trying: tail -c +17 CL45737-2024.02.14-21.18.SGReplay | zcat | ~/go/bin/protoscope | less
For some replays this was at least extracting the map and player names:
3: 7.974551e-24i32 # 0x191a4010i32
1: {
3: {
2: {"BrokenCrown"}
3: 1566072066
6: 1
}
}
5:EGROUP
2: 64
3: {
1: {
37: {
1: {
1: 6518020192259623528
2: -5630106735107133817
}
2: 1
3: {"Sturgeon"}
}
}
}
2: 16
8: 26
1:EGROUP
1: {
15: {
3: 1
4: 2952722564
}
}
5: 16
8: 26
4:EGROUP
1: {
37: {
1: {
1: -4128416236917469546
2: -5721755048076107725
}
2: 2
3: {"Pox"}
}
}
2: {`401a0e0a0c7a0a18022084f1fbff0a28`}
`010910401a050a038a01000b08f00510011a040a0222000b08fd0510021a040a0222000b08bb0710`
# rest of output is similar unstructured binary
But for other replays it would cut to the binary earlier, not finding the map and player names. From inspecting the files I can see those strings are in there, but protoscope just isn't seeing the structure right. I'm not well-versed in protobuf so don't really know what the issue could be here - let me know if you make any progress!
Heya, tried the script right now but <5% of my replays got renamed (only 19th-22th of February nothing prior). Does it not work for prior matches? Or what is the reason for this?
While Debugging its clear that the find_match function simply does not find a match because they dont appear in the "matches" dataframe
Edit: The matches API query is paged, can just iterate through all of the pages
Edit2: Might be important to rename some opponents if they have special character in their name that are not allowed in your OS
@WinckelData: Thanks for your comment!
When I wrote the script, only the first page of player matches was available. You're right, they've now added a page parameter to that endpoint so we should be able to go back as far as we need. I'll post an updated version soon :)
I haven't had any issues with filenames yet, but you're right that it could be a problem.
@acarapetis awesome work!
Still getting some errors for the username "And It went like | Moon" due to the vertical bar.
your sanitize_filename function just looks at the start of the string and deletes just the first letter.
FYI, the forbidden printable ASCII characters for filenames are:
Linux/Unix:
- / (forward slash)
Windows:
- < (less than)
- > (greater than)
- : (colon - sometimes works, but is actually NTFS Alternate Data Streams)
- " (double quote)
- / (forward slash)
- \ (backslash)
- | (vertical bar or pipe)
- ? (question mark)
- * (asterisk)
May i suggest something like
def sanitize_filename(filename: str) -> str:
# Forbidden characters in filename
pattern = r'[<>:"/\\|?*]'
# Replace the characters with an empty string
sanitized_filename = re.sub(pattern, "", filename)
return sanitized_filename
@WinckelData Had a typo in the regex, fixed it in revision 3 but looks like you got in quick :)
Your approach is better though - should give more consistent results and I'm pretty sure we don't need to worry about any other platforms. I'll publish a new revision.
@Fafa87 worked out the replay files, there's an extra layer on top of the protobuf - after unzipping, what you have is a sequence of length-prefixed protobuf messages. (e.g. if the first byte is 0x30, the next 48 bytes are a protobuf message, with the byte following being the next length marker). Looks like we could get map name and player nicknames pretty reliably with that, anything more will be a struggle though.
Map names would be a great addition to the renaming! 👍
I'm now getting map names out of the replays successfully, as well as double-checking the player nicknames. It's grown a bit big to be a gist now, so I've published it as a python package - https://github.com/acarapetis/shroudstone
Awesome Work, Haven't tried it yet only went a bit over the code on my phonr but I would highly appreciate a way to customize the renaming schema If that is not already implemented
Read my mind, working on custom format strings already :)
Great work here. I tried this out on linux+proton and it just worked by doing the pip install and just running it.
I did get some errors of games not found but I think its because the replays were against ai so they didn't have a game logged on stormgateworld.
output:
ERROR No match found for CL45737-2024.02.18-20.00.SGReplay in stormgateworld match history. renamer.py:86
ERROR No match found for CL45737-2024.02.18-20.03.SGReplay in stormgateworld match history. renamer.py:86
Silly question but you unzipped them first, right?