Local collection report for Cities:Skylines assets

LocalCollectionReport.py

import json
import os
import re
import requests
import datetime
import sys
import webbrowser
from time import sleep
from random import uniform
from os.path import exists
from functools import partial
from bs4 import BeautifulSoup
from PySide6 import QtCore, QtWidgets, QtGui
from pprint import pprint
from urllib.parse import urlparse

#######################################################################################################
##              USER EDITABLE constants

#####     where currently loaded .crp assets are stored for play 
LOCAL_CITIES_DATA_PATH = os.getenv('LOCALAPPDATA') + "\\Colossal Order\\Cities_Skylines\\Addons\\Assets\\"

#####     path to the local dir containing colle json
LOCAL_COLLE_PATH = LOCAL_CITIES_DATA_PATH + "yourCollectionFolder"

#####    delay between collection page scrapes
SLEEP_TIME = (0.5, 1)

########################################################################################################
##              constants
# which column represents the item ID
ID_COL = 4
# transparency of column backgrounds
COL_OPACITY = 45
# error col color
COL_ERR_COLOR = QtGui.QColor(255, 0, 0, COL_OPACITY * 2)
# warn col color
COL_WARN_COLOR = QtGui.QColor(255, 200, 0, COL_OPACITY * 2)
# column colors
COL_COLOR = QtGui.QColor(144, 144, 144, COL_OPACITY)

# steamcommunity file details URL path
FILE_DETAILS_URL = 'https://steamcommunity.com/sharedfiles/filedetails/?id='

########################################################################################################

class CollectionsManagerWidget(QtWidgets.QWidget):

# binding for the treeview doubleclick
    @QtCore.Slot(QtCore.QModelIndex)
    def on_doubleclick_item(self, index):
        for idx in self.itemsView.selectedIndexes():
            if(self.itemsView.model().itemFromIndex(idx).column() == ID_COL):
                url = (FILE_DETAILS_URL + self.itemsView.model().itemFromIndex(idx).text())
                webbrowser.open(url)


# create treeview columns
    def create_col(self, i, key, back_color, to_str = False):
        col = QtGui.QStandardItem((i[key] if not to_str else str(i[key]))) 
        col.setBackground(back_color)
        return col
    
# create treeview rows
    def create_rows(self, items):
        for i in items:
            item = [self.create_col(i, "collection_id", COL_COLOR),
                    self.create_col(i, "collection_name", COL_COLOR),
                    self.create_col(i, "exists_locally", COL_COLOR if i["exists_locally"] else COL_ERR_COLOR, True),
                    self.create_col(i, "not_collected", COL_COLOR if not i["not_collected"] else COL_WARN_COLOR, True),
                    self.create_col(i, "item_id", COL_COLOR),
                    self.create_col(i, "item_title", COL_COLOR),
                    self.create_col(i, "item_author", COL_COLOR)]
            
            self.model.appendRow(item)

# evaluate a single item entry within the workshop page. 
#   returns a dict representing the specific collection item.            
    def evaluate_item(self, local_assets, item, collection_id, collection_name):
        # extract the item's ID from the query component of URL
        ws_url = item.find(class_='workshopItem').a.get('href')
        u = urlparse(ws_url)
        item_id = self.id_pattern.findall(u.query)[0]

        # get the collectionItemDetails element from page, then extract relevant data
        ws_details = item.find(class_='collectionItemDetails')
        item_title = ws_details.find(class_='workshopItemTitle').get_text()
        item_author = self.author_pattern.findall(ws_details.find(class_='workshopItemAuthor').get_text())[0]

        # evaluate whether there's a corresponding dir for this item under local_assets
        exists_locally = True if item_id in local_assets else False
        if item_id in local_assets:
            local_assets.remove(item_id) # remove this processed item from list of loc.assets to process.

        # format this item's properties as dict
        this_item = {"collection_id": collection_id, "collection_name": collection_name, 
                       "item_id": item_id, "item_title": item_title, 
                       "item_author": item_author, "exists_locally": exists_locally, "not_collected": False }
        return this_item


# get and evaluate a workshop collections page.
#   returns a list of dicts where each entry is a scraped collection item.
    def evaluate_collection(self, collection):
        colle_items = [] # list of dict

        id = collection["id"]
        loc_path = collection["local_path"]

        # request and download the page data for this collection
        url = ('https://steamcommunity.com/sharedfiles/filedetails/?id=' + id)
        print("Fetching page " + url)
        page_download = requests.get(url)
        print("Got status code " + str(page_download.status_code))

        # parse the downloaded page data as HTML
        page = BeautifulSoup(page_download.content, 'html.parser')

        # get every collectionItem element from the page
        items = page.find_all(class_='collectionItem')

        # walk the local dir path to gather directory names (local assets)
        loc_assets = list()
        print("Checking local asset directories under " + LOCAL_CITIES_DATA_PATH + loc_path)
        for root, dirs, files in os.walk(LOCAL_CITIES_DATA_PATH + loc_path, topdown=False):
            for pathname in dirs:
                if pathname[0].isdigit():
                    loc_assets.append(pathname)
                    print("Adding local asset directory: " + pathname)
        
        # eval each collectionItem from the page, then create an items table
        for item in items:
            this_item = self.evaluate_item(loc_assets, item, id, collection["name"])
            colle_items.append(this_item)

        for rem_item in loc_assets:
            # directory should begin with a number, else it's a category folder.
            this_rem_item = {'url':(FILE_DETAILS_URL + rem_item), 
                       'collection_id':id, 
                       'collection_name':collection["name"], 
                       'item_title':"", 
                       'item_author':"", 
                       'item_id':rem_item, 
                       'exists_locally':True, 
                       'not_collected':True
                       }
            colle_items.append(this_rem_item)

        return colle_items
            

    def __init__(self):
        super().__init__()

        # precompile regex patterns for scraping the collection pages
        self.id_pattern = re.compile(r'\d+')
        self.author_pattern = re.compile(r'(?:Created by[\t]+)(.+)(?:\n*)')

        # add outer container, items table
        self.outerLayout = QtWidgets.QVBoxLayout(self)
        self.itemsView = QtWidgets.QTreeView()
        # establish treeview model
        self.model = QtGui.QStandardItemModel()
        self.model.setColumnCount(7)
        self.model.setHorizontalHeaderLabels(["Collection ID", "Collection Name", "Exists Locally", "Not in Collection", "Item ID", "Name", "Author"])
        self.itemsView.setModel(self.model)

        # blocks double-click editable fields in the treeview
        #self.itemsView.setEditTriggers(QtWidgets.QAbstractItemView.NoEditTriggers)

        # attach itemsview to outer
        self.outerLayout.addWidget(self.itemsView)

        the_items = [] # list of dicts

        json_file = LOCAL_COLLE_PATH + "\\colle.json"
        with open(json_file) as json_data:
            data = json.load(json_data)

            for info in data['collections']:
                sleep(uniform(SLEEP_TIME[0], SLEEP_TIME[1]))
                the_items = the_items + self.evaluate_collection(info)

        # create rows for all items
        self.create_rows(the_items)
        self.itemsView.doubleClicked.connect(self.on_doubleclick_item)
        self.itemsView.setSortingEnabled(True)


if __name__ == "__main__":
    app = QtWidgets.QApplication([])

    widget = CollectionsManagerWidget()
    widget.setWindowTitle("Asset Report")
    widget.resize(800, 600)
    widget.show()

    sys.exit(app.exec())