Last active
March 21, 2021 23:07
-
-
Save DustinAlandzes/b765d54ba13b8ee28d7871dc207e7712 to your computer and use it in GitHub Desktop.
Copy of slippi-graph.ipynb
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "nbformat": 4, | |
| "nbformat_minor": 0, | |
| "metadata": { | |
| "colab": { | |
| "name": "Copy of slippi-graph.ipynb", | |
| "private_outputs": true, | |
| "provenance": [], | |
| "collapsed_sections": [], | |
| "toc_visible": true, | |
| "machine_shape": "hm", | |
| "authorship_tag": "ABX9TyMZdEVpNFuBttC+a1Ittz8j", | |
| "include_colab_link": true | |
| }, | |
| "kernelspec": { | |
| "display_name": "Python 3", | |
| "name": "python3" | |
| } | |
| }, | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "view-in-github", | |
| "colab_type": "text" | |
| }, | |
| "source": [ | |
| "<a href=\"https://colab.research.google.com/gist/DustinAlandzes/b765d54ba13b8ee28d7871dc207e7712/copy-of-slippi-graph.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "pTeKDnpr4fwS" | |
| }, | |
| "source": [ | |
| "https://py-slippi.readthedocs.io/en/latest/source/slippi.html\n", | |
| "\n", | |
| "https://github.com/BrodyVoth/slippi-cumulative-stats/blob/master/slippi-stats.js" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "23xpBNf7yEA4" | |
| }, | |
| "source": [ | |
| "!pip install py-slippi\n", | |
| "!pip install requests\n", | |
| "\n", | |
| "!pip install requests-cache\n", | |
| "import requests_cache\n", | |
| "requests_cache.install_cache('requests_cache')\n", | |
| "# https://github.com/reclosedev/requests-cache#usage-example\n", | |
| "# requests_cache helps prevent unneeded waiting for the data to re-download\n", | |
| "\n", | |
| "# upgrade ipython and ipykernel so I can use async/await\n", | |
| "!pip install ipython ipykernel --upgrade" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "zgJ1BjbcyOGC" | |
| }, | |
| "source": [ | |
| "from typing import Text\n", | |
| "import requests\n", | |
| "import os\n", | |
| "from pathlib import Path\n", | |
| "\n", | |
| "SLP_DIRECTORY_NAME = '/content/slp'\n", | |
| "Path(SLP_DIRECTORY_NAME).mkdir(parents=True, exist_ok=True)\n", | |
| " \n", | |
| "# download a slp file for testing\n", | |
| "# TODO: https://docs.python.org/3/library/pathlib.html\n", | |
| "\n", | |
| "# with open(f'{SLP_DIRECTORY_NAME}/test.slp', 'wb') as f:\n", | |
| "# response = requests.get(\"https://spaceanimalz.com/f/8c72f820cf154b4581de/?raw=1\")\n", | |
| "# f.write(response.content)\n", | |
| "\n", | |
| "# test_game = Game(f'./{SLP_DIRECTORY_NAME}/test.slp')\n", | |
| "# duration_in_minutes = test_game.metadata.duration // 60 // 30\n", | |
| "# print(f\"duration: {duration_in_minutes} minutes\") # in minutes\n", | |
| "# print(test_game.start.stage) # stage is in start attribute" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "AeM6CqJGfQED" | |
| }, | |
| "source": [ | |
| "# cursed stuff I'm doing to download slippi files from seafile\n", | |
| "\n", | |
| "generate a share link (I use an expiration date) and put it in the share_link variable below\n", | |
| "\n", | |
| "" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "hxlrGJ6sGekL" | |
| }, | |
| "source": [ | |
| "from typing import Any, Dict, Optional, Text, Set\n", | |
| "from pathlib import Path\n", | |
| "import os\n", | |
| "from multiprocessing import Pool\n", | |
| "\n", | |
| "class SeafileException(Exception):\n", | |
| " \"\"\" to make it clear this is expected if you are not using a shared folder \"\"\"\n", | |
| "\n", | |
| "\n", | |
| "def get_list_of_file_links_from_share_link(share_link: Text, path: Optional[Text] = None) -> Set[Text]:\n", | |
| " \"\"\" takes in a seafile share link, \n", | |
| " outputs a list of local file paths (pathlib objects?) \n", | |
| "\n", | |
| " make sure the share link has the ending forward slash (/)\n", | |
| " \n", | |
| " use the api to list the files and directories in the shared folder\n", | |
| "\n", | |
| " based on https://github.com/mperreir/share_link_dl\n", | |
| " \"\"\"\n", | |
| "\n", | |
| " # create the api link from the share link\n", | |
| " api_link: Text = f\"{share_link.replace('/d/', '/api/v2.1/share-links/')}dirents/\"\n", | |
| " if path:\n", | |
| " api_link = f\"{api_link}?path={requests.utils.quote(path)}\"\n", | |
| " print(api_link)\n", | |
| "\n", | |
| " # do a GET request to the api link\n", | |
| " response: requests.Response = requests.get(api_link)\n", | |
| " try:\n", | |
| " response.raise_for_status()\n", | |
| " except requests.HTTPError as e:\n", | |
| " status_code: int = e.response.status_code\n", | |
| " if status_code == 400:\n", | |
| " raise SeafileException(response.json()['error_msg'])\n", | |
| " else:\n", | |
| " raise\n", | |
| "\n", | |
| " file_urls: List[Text] = []\n", | |
| " for file_or_directory in response.json()['dirent_list']:\n", | |
| " is_directory = file_or_directory['is_dir']\n", | |
| " if is_directory:\n", | |
| " # call this function recursively with the path parameter set to the 'folder_path' key\n", | |
| " folder_path = file_or_directory['folder_path']\n", | |
| " file_urls.extend(download_seafile_directory_from_share_link(share_link, folder_path))\n", | |
| " else:\n", | |
| " # add the file url to the set\n", | |
| " file_path: Text = file_or_directory['file_path']\n", | |
| " file_url: Text = \"{}files/?p={}&dl=1\".format(share_link, requests.utils.quote(file_path))\n", | |
| " file_urls.append(file_url)\n", | |
| "\n", | |
| " return file_urls\n", | |
| "\n", | |
| "share_link: Text = \"https://spaceanimalz.com/d/96a2f7687715427ead65/\"\n", | |
| "slp_urls = get_list_of_file_links_from_share_link(share_link)\n", | |
| "print(slp_urls)" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "PIFPxR_xvKwn" | |
| }, | |
| "source": [ | |
| "async def download_and_save(url: Text) -> Path:\n", | |
| " destination_path = Path('/content/slp', f'./{url.replace(\"&dl=1\", \"\").split(\"?p=\")[1]}')\n", | |
| "\n", | |
| " # create parent directory if it doesn't exist\n", | |
| " destination_path.parent.mkdir(parents=True, exist_ok=True)\n", | |
| "\n", | |
| " # only download it if it isn't already downloaded\n", | |
| " if destination_path.is_file():\n", | |
| " print(f\"Already downloaded {desination_path}\")\n", | |
| " return destination_path\n", | |
| "\n", | |
| " print(f\"Downloading {url} to {destination_path}\")\n", | |
| " response = requests.get(url)\n", | |
| " with open(destination_path, \"wb\") as f:\n", | |
| " f.write(response.content)\n", | |
| " return destination_path\n", | |
| "\n", | |
| "for url in slp_urls:\n", | |
| " await download_and_save(url)" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "mKjkXJr1HXYp" | |
| }, | |
| "source": [ | |
| "import glob\n", | |
| "from typing import Any, Dict, List, Text\n", | |
| "from slippi import Game\n", | |
| "\n", | |
| "import pandas as pd\n", | |
| "\n", | |
| "STAGE_MAPPING = {\n", | |
| " 2: 'Fountain of Dreams',\n", | |
| " 3: 'Pokemon Stadium',\n", | |
| " 8: 'Yoshi\\'s Story',\n", | |
| " 28: 'Dream Land N64',\n", | |
| " 31: 'Battlefield',\n", | |
| " 32: 'Final Destination'\n", | |
| "}\n", | |
| "\n", | |
| "# todo: all stages, from friendlies. create dict from int enum\n", | |
| "\n", | |
| "async def load_slp_file(path: Text) -> Dict[Text, Any]:\n", | |
| " \"\"\" given a path, loads game into memory and returns a dictionary with the information \"\"\"\n", | |
| " with open(path, 'rb') as f:\n", | |
| " game = Game(f)\n", | |
| " return {'duration': game.metadata.duration / 60 / 60, 'date': game.metadata.date, 'Stage': STAGE_MAPPING[game.start.stage],\n", | |
| " 'Version': str(game.start.slippi.version)}\n", | |
| "\n", | |
| "\n", | |
| "games: List[Dict[Text, List[Any]]] = []\n", | |
| "for path in glob.glob(directory + \"**/*.slp\", recursive=True):\n", | |
| " try:\n", | |
| " game = await load_file(path)\n", | |
| "\n", | |
| " # todo: if any of them fail, don't append any\n", | |
| "\n", | |
| " games['duration'].append(game['duration'])\n", | |
| " games['date'].append(game['date'])\n", | |
| " # games['character_1'].append(game.metadata.players[0])\n", | |
| " # games['character_2'].append(game.metadata.players[1])\n", | |
| " games['Stage'].append(game['Stage'])\n", | |
| " games['Version'].append(str(game['Version'])\n", | |
| " # games['character_1_win'].append(None)\n", | |
| " # games['character_2_win'].append(None)\n", | |
| " # print(game.metadata.players[0].netplay.code)\n", | |
| " # print(game.metadata.players[1].netplay.code)\n", | |
| " # print(game.metadata.players[1].netplay.name)\n", | |
| " # print(game.metadata.players[1].characters)\n", | |
| " except Exception as e:\n", | |
| " # don't stop parsing because one file isn't working\n", | |
| " print(f\"There was a problem parsing {path}\")\n", | |
| " # delete it? could be that it just downloaded badly and I need to handle large files better\n", | |
| " print(e)\n", | |
| "\n", | |
| "print(f\"Number of games: {len(games)}\")" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "BOqyMBjWsSwB" | |
| }, | |
| "source": [ | |
| "for key in list_of_games.keys():\n", | |
| " print(key, len(list_of_games[key]))" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "ENeUMsTIroyT" | |
| }, | |
| "source": [ | |
| "df = pd.DataFrame(list_of_games)\n", | |
| "df.to_parquet('df.parquet.gzip', compression='gzip')\n", | |
| "pd.read_parquet('df.parquet.gzip')" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "YJQ8RZzkHZlK" | |
| }, | |
| "source": [ | |
| "# group by day: game count, duration sum, win loss ratio, \n", | |
| "# per character, duration per stage, per matchup\n", | |
| "\n", | |
| "import altair as alt\n", | |
| "\n", | |
| "alt.data_transformers.disable_max_rows() # MaxRowsError\n", | |
| "minutes_per_day_and_stages_chart = alt.Chart(df).properties(\n", | |
| " width=750,\n", | |
| " height=500\n", | |
| ").transform_aggregate(\n", | |
| " daily_minutes = 'sum(duration)',\n", | |
| " groupby=['date', 'Stage']\n", | |
| " ).mark_bar().encode(\n", | |
| " x = alt.X('yearmonthdate(date):O'),\n", | |
| " y = alt.Y('daily_minutes:Q'),\n", | |
| " color = 'Stage:N'\n", | |
| ")\n", | |
| "\n", | |
| "\n", | |
| "minutes_per_day_and_stages_chart" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "metadata": { | |
| "id": "rlbXZ2EbCt0k" | |
| }, | |
| "source": [ | |
| "" | |
| ], | |
| "execution_count": null, | |
| "outputs": [] | |
| } | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment