Fallout 76 IconSortingTags Strings Mod Merger Script

merge.py

# ############### HOW TO USE THIS FILE ################
#
# NOTE: Requires Python 3.9+ and the package `xmltodict` installed.
#
# 1. Drop this file in the base level directory of the "Tagged Plans - XML for xTranslator" mod
# 2. Open xTranslator, load 76's ESP/ESM, and process the IconSortingTag's header rules
#   - import the fonts to your ba2s too
# 3. Export IconSortingTag's changes as an XML translation
#   - File -> Export Translation -> XML files
#   - Name file `full_icon_dump.xml` and place in directory of the "Tagged Plans" mod
# 4. Run this Python script.
#   - Script merges IconSortingTag's changes with the Tagged Plans changes as:
#     `🔫 [Ops] Cool Gun From Daily Ops`
# 5. Import merged XML as a translation
#   - File -> Import Translation -> XML Files (xTranslator)
#       - Select `out.xml`
#       - Overwrite should be set to "Everything"
#       - Mode should be set to "Use FormID References"
# 6. Save Strings
#   - File -> Finalize STRINGS
#       - Overwrite any existing strings files
# 7. Close xTranslator and start game!
#
# NOTE: This file also supports merging other mods as well:
#
# --- Clear descriptions and scrap names - https://www.nexusmods.com/fallout76/mods/1814 ---
# 1. Open xTranslator and freshly load 76's ESP/ESM
# 2. Import strings as translation
#   - Tools -> Load .Strings as translation
# 3. Export as XML
#   - File -> Export Translation -> XML files
#       - Only strings with source != dest
#   - Name file `clear_descriptions.xml` and place in directory of the "Tagged Plans" mod
# 4. Follow step 4 and onward in the instructions above!
#
# --- Tagged Rare Plans and Apparel - https://www.nexusmods.com/fallout76/mods/1409 ---
# 1. Download the two optional files called `Apparel Tags` and `Plan Tags by Source`
# 2. Extract the folders from these zip files to the base level directory of the
#    "Tagged Plans - XML for xTranslator" mod
# 3. Follow all original instructions above!
#   - Beyond merging with the `IconSortingTag` mod, it will also merge tags with the
#     `Tagged Plans` mod
#       - This mod will be prioritized over the `Tagged Plans` mod's tags
#
# --- Quizzless Apalachia - https://www.nexusmods.com/fallout76/mods/305 ---
# 0. THIS SHOULD BE DONE LAST
# 1. Once you have finalized all other string changes, import the STT file.
#   - I went thru and validated this doesn't impact anything any of the other strings modify.
#   - File -> Import Translation -> STT Dictionary
#       - Overwrite should be set to "Everything"
#       - Mode should be set to "Use FormID References"
# 2. Follow Steps 6 and onward in the topmost set of instructions!
# #####################################################

# ############# CONSTANTS - CHANGE THESE ##############
# ------------------ IconSortingTag -------------------
XTRANSLATE_ICONS_MOD_DUMP = "full_icon_dump.xml"
# -------- Clear descriptions and scrap names ---------
CLEARER_DESCRIPTIONS_NAME = "clear_descriptions.xml"
# -------- Tagged Plans - XML for xTranslator ---------
ALL_CATEGORIES_FILE_NAME = "SeventySix_en_en.xml"
EXTENDED_TAGS_FILE_NAME = "ExtendedTag_en.xml"
SILO_HOLO_NAME = "SiloHolotape_en.xml"
# ----------- Tagged Rare Plans and Apparel -----------
PLANS_DIR_PATH = "./Plans by Source"
APPAREL_DIR_PATH = "./Apparel Tags"
# #####################################################


import re  # noqa
import logging  # noqa
import xmltodict  # noqa
from functools import reduce  # noqa
from pathlib import Path  # noqa

logging.basicConfig(
    level=logging.INFO,
    format="{asctime} [{levelname}] {message}",
    style="{",
    datefmt="%Y-%m-%d %H:%M:%S",
)

log = logging.getLogger()

plan_file = {}
recipe_file = {}
icon_file = {}
renamed_file = {}
tagged_replacements = [
    # BREWING.xml
    ("#BREWING#", "[Brew]"),
    # EVENT.xml
    ("#FASNACHT#", "[Fas]"),
    ("#INVADERS#", "[Alien]"),
    ("#GIFTS-20#", "[G20]"),
    ("#GIFTS-21#", "[G21]"),
    ("#HALLOWEEN#", "[Hall]"),
    ("#MOTHMAN#", "[MM]"),
    ("#MW#", "[MW]"),
    ("#PAILS#", "[Pail]"),
    ("#PAILS-20#", "[Pail20]"),
    ("#PAILS-21#", "[Pail21]"),
    # FREERANGE.xml
    ("#FREE RANGE#", "[FR]"),
    # GRAHM.xml
    ("#GRAHM#", "[G]"),
    # LEGACY.xml
    ("#LEGACY#", "[Leg]"),
    # NW.xml
    ("#NW#", "[NW]"),
    # RARE.xml
    ("#RARE#", "[$$$]"),
    # SCOUTS.xml
    ("#SCOUTS#", "[Scout]"),
    # UNCOMMON.xml
    ("#UNCOMMON#", "[$]"),
    # OPS.xml
    ("#OPS#", "[Ops]"),
    # ExtendedTag_en.xml
    ("#RARE_ITEM#", "[Rare]"),
]
ph_enabled_tagged_replacements = [
    "#INVADERS#",
    "#GIFTS-20#",
    "#GIFTS-21#",
    "#HALLOWEEN#",
    "#MOTHMAN#",
    "#MW#",
    "#PAILS#",
    "#PAILS-20#",
    "#PAILS-21#",
    "#FREE RANGE#",
    "#GRAHM#",
    "#OPS#",
]

plan_hunter_ascii = ["¬", "±", "¤", "¢"]
plan_hunter_replacements = [
    # Apparel Tags - Ultra Rare.xml
    ("¬¬¬", "[!!!]"),
    # Apparel Tags - Rare.xml
    ("¬", "[$$$]"),
    # Plan Tags - Colossal Problem.xml
    # -No replacements-
    # ("[CP]", "[CP]"),
    # Plan Tags - Daily Ops.xml
    ("[DOps]", "[Ops]"),
    # Plan Tags - Encryptid.xml
    ("[ENC]", "[Cryp]"),
    # Plan Tags - Fasnacht.xml
    ("[FASN]", "[Fas]"),
    # Plan Tags - Free Range.xml
    # -No replacements-
    # ("[FR]", "[FR]"),
    # Plan Tags - General Rare.xml
    # -No replacements - handled by Apparel tag above-
    # ("¬", "[$$$]"),
    # Plan Tags - General Uncommon Valueable.xml
    ("±", "[$]"),
    # Plan Tags - Grahm Unique Plans.xml
    ("[GRAHM]", "[G]"),
    # Plan Tags - Holiday Gifts.xml
    ("[GIFTS]", "[Gift]"),
    # Plan Tags - Illegal or Unobtainable.xml
    ("¤", "[Hack]"),
    # Plan Tags - Invaders from Beyond.xml
    ("[ALIENS]", "[Alien]"),
    # Plan Tags - Meat Week.xml
    ("[MEAT]", "[Meat]"),
    # Plan Tags - Moleminer Pails.xml
    ("[PAILS]", "[Pail]"),
    # Plan Tags - Moonshine Jamboree.xml
    # -No replacements-
    # ("[MJ]", "[MJ]"),
    # Plan Tags - Most Wanted.xml
    ("[MW]", "[Wanted]"),
    # Plan Tags - Mothman Equinox Plans.xml
    ("[MOTH]", "[Moth]"),
    # Plan Tags - Nuka World on Tour.xml
    # -No replacements-
    # ("[NWOT]", "[NWOT]"),
    # Plan Tags - Pitt Expeditions.xml
    ("[PITT]", "[Pitt]"),
    # Plan Tags - Project Paradise.xml
    # -No replacements-
    # ("[PP]", "[PP]"),
    # Plan Tags - Radiation Rumble.xml
    # -No replacements-
    # ("[RR]", "[RR]"),
    # Plan Tags - Seismic Activity Plans.xml
    # -No replacements-
    # ("[SA]", "[SA]"),
    # Plan Tags - Spin the Wheel.xml
    # -No replacements-
    # ("[StW]", "[StW]"),
    # Plan Tags - Spooky Treat Bags.xml
    ("[BAGS]", "[HWN]"),
    # Plan Tags - Test your Metal.xml
    ("[METAL]", "[Metal]"),
    # Plan Tags - Tunnel of Love.xml
    # -No replacements-
    # ("[ToL]", "[ToL]"),
    # Plan Tags - Untradable - Just Read Them.xml
    ("[¢]", "[X]"),
]

preferred_override = {"[!!!]": "[$$$]", "[$$$]": "[$]"}

if not Path(XTRANSLATE_ICONS_MOD_DUMP).is_file():
    log.error("Missing IconSortingTag's Dumped XML")
    exit()


def clean_malformed_xml(malformed_xml):
    malformed_xml = "\n".join([line for line in malformed_xml.split("\r\n") if line.strip() != ""])
    malformed_xml = re.sub(r"\n([^ <])", r"\\n\g<1>", malformed_xml)
    malformed_xml = re.sub(r"\n([ ])(?! )", r"\\n\g<1>", malformed_xml)
    malformed_xml = re.sub(r"\n([ ]{2,2})(?!<| )", r"\\n\g<1>", malformed_xml)
    malformed_xml = re.sub(r"\n(</Source>)", r"\g<1>", malformed_xml)
    malformed_xml = re.sub(r"\n( </Source>)", r"\g<1>", malformed_xml)
    malformed_xml = re.sub(r"\n(</Dest>)", r"\g<1>", malformed_xml)
    malformed_xml = re.sub(r"\n( </Dest>)", r"\g<1>", malformed_xml)
    malformed_xml = malformed_xml.replace("\xBE\b", "")
    return malformed_xml


with open(XTRANSLATE_ICONS_MOD_DUMP, "rb") as r:
    log.info("Parsing Icons File...")
    string_file = clean_malformed_xml(r.read().decode("utf-8"))
    icon_file = xmltodict.parse(string_file, encoding="utf-8")
    log.info("Done!")

with open(SILO_HOLO_NAME, "rb") as r:
    log.info("Parsing Silo Holo File...")
    string_file = clean_malformed_xml(r.read().decode("utf-8"))
    holo_file = xmltodict.parse(string_file, encoding="utf-8")["SSTXMLRessources"]["Content"][
        "String"
    ]
    holo_file_blob = {holo_file["@sID"]: holo_file["Dest"]}
    log.info("Done!")

if not Path(f"./{ALL_CATEGORIES_FILE_NAME}").is_file():
    log.error("ERROR: Missing all categories 'Tagged Plans' file.")
    exit()

with open(ALL_CATEGORIES_FILE_NAME, "rb") as r:
    log.info("Parsing Tagged Plans' all categories file...")
    renamed_file = xmltodict.parse(r.read())
    log.info("Done!")

valid_renames = {
    item_blob["@sID"]: re.sub(r"# (.*)", "#", item_blob["Dest"])
    for item_blob in renamed_file["SSTXMLRessources"]["Content"]["String"]
    if item_blob["Dest"]
}

extended_valid_renames = {}
if Path(f"./{EXTENDED_TAGS_FILE_NAME}").is_file():
    log.info("Found extended tags file! Parsing file...")
    with open(EXTENDED_TAGS_FILE_NAME, "rb") as r:
        extended_renamed_file = xmltodict.parse(r.read())

    extended_valid_renames = {
        item_blob["@sID"]: re.sub(r"# (.*)", "#RARE_ITEM#", item_blob["Dest"])
        for item_blob in extended_renamed_file["SSTXMLRessources"]["Content"]["String"]
        if item_blob["Dest"]
    }
    log.info("Done!")


def rfind_by_list(query_string, search_list):
    for search_string in search_list:
        if query_string.rfind(search_string) > -1:
            return query_string.rfind(search_string)
    return -1


plan_hunter_valid_renames = {}
if Path(APPAREL_DIR_PATH).is_dir() and Path(PLANS_DIR_PATH).is_dir():
    log.info("Found Plan Collector Dirs! Parsing files...")
    tagged_replacements = [
        (blob[0], blob[1]) if blob[0] not in ph_enabled_tagged_replacements else (blob[0], "")
        for blob in tagged_replacements
    ]
    plan_hunter_merged_xml = {}
    for p in Path(APPAREL_DIR_PATH).rglob("*"):
        if plan_hunter_merged_xml:
            string_list = xmltodict.parse(p.read_bytes())["SSTXMLRessources"]["Content"]["String"]
            string_list = [string_list] if isinstance(string_list, dict) else string_list
            plan_hunter_merged_xml["SSTXMLRessources"]["Content"]["String"].extend(string_list)
        else:
            plan_hunter_merged_xml = xmltodict.parse(p.read_bytes())
    for p in Path(PLANS_DIR_PATH).rglob("*"):
        if plan_hunter_merged_xml:
            string_list = xmltodict.parse(p.read_bytes())["SSTXMLRessources"]["Content"]["String"]
            string_list = [string_list] if isinstance(string_list, dict) else string_list
            plan_hunter_merged_xml["SSTXMLRessources"]["Content"]["String"].extend(string_list)
        else:
            plan_hunter_merged_xml = xmltodict.parse(p.read_bytes())
    plan_hunter_valid_renames = {
        item_blob["@sID"]: item_blob["Dest"][: item_blob["Dest"].rfind("]") + 1]
        if item_blob["Dest"].rfind("]") > 0
        else item_blob["Dest"][: rfind_by_list(item_blob["Dest"], plan_hunter_ascii) + 1]
        for item_blob in plan_hunter_merged_xml["SSTXMLRessources"]["Content"]["String"]
        if item_blob["Dest"]
    }
    log.info("Done!")

clearer_descriptions_renames = {}
if Path(f"./{CLEARER_DESCRIPTIONS_NAME}").is_file():
    log.info("Found clearer descriptions file! Parsing file...")
    with open(CLEARER_DESCRIPTIONS_NAME, "rb") as r:
        clearer_descriptions_file = xmltodict.parse(r.read())

    clearer_descriptions_renames = {
        item_blob["@sID"]: item_blob["Dest"]
        for item_blob in clearer_descriptions_file["SSTXMLRessources"]["Content"]["String"]
        if item_blob["Dest"]
    }
    log.info("Done!")

log.info("Cleaning data to be merged...")
cleaned_renames = valid_renames | extended_valid_renames

replaced_renames = {
    item_id: reduce(
        lambda string_name, replacement_blob: string_name.replace(*replacement_blob),
        tagged_replacements,
        string_name,
    )
    for item_id, string_name in cleaned_renames.items()
}
replaced_renames = {
    item_id: string_name for item_id, string_name in replaced_renames.items() if string_name
}


plan_hunter_replaced_renames = {
    item_id: reduce(
        lambda string_name, replacement_blob: string_name.replace(*replacement_blob),
        plan_hunter_replacements,
        string_name,
    )
    for item_id, string_name in plan_hunter_valid_renames.items()
}
plan_hunter_replaced_renames = {
    item_id: string_name
    for item_id, string_name in plan_hunter_replaced_renames.items()
    if string_name
}
log.info("Done!")

# Just to make checking out file's validity more sane
reduced_renames = (
    replaced_renames | clearer_descriptions_renames | holo_file_blob | plan_hunter_replaced_renames
)
reduced_icon_file = [
    item
    for item in icon_file["SSTXMLRessources"]["Content"]["String"]
    if item["@sID"] in reduced_renames
]
icon_file["SSTXMLRessources"]["Content"]["String"] = reduced_icon_file

log.info("Replacing strings in data block...")
for item in icon_file["SSTXMLRessources"]["Content"]["String"]:
    collisions = [
        item_list.get(item["@sID"])
        for item_list in [
            replaced_renames,
            clearer_descriptions_renames,
            holo_file_blob,
            plan_hunter_replaced_renames,
        ]
        if item_list.get(item["@sID"])
    ]
    if len(collisions) > 1:
        log.warning(
            f"Collision detected with item by ID '{item['@sID']}'\n -> "
            f"'{item['Dest']}'\n -- {' / '.join(collisions)}"
        )

    if item["@sID"] in holo_file_blob:
        item["Dest"] = holo_file_blob[item["@sID"]]

    if item["@sID"] in replaced_renames and item["@sID"] in plan_hunter_replaced_renames:
        item["Source"] = item["Dest"]
        ph_rename = plan_hunter_replaced_renames[item["@sID"]]
        tp_rename = replaced_renames[item["@sID"]]
        substring = ph_rename
        if tp_rename == preferred_override.get(ph_rename):
            pass
        elif ph_rename == preferred_override.get(tp_rename):
            substring = tp_rename
        elif tp_rename not in substring:
            substring = ph_rename + tp_rename
        item["Dest"] = re.sub(
            r"([^\x00-\x1F\x21-\x7F]+)(.*)",
            rf"\1{substring} \2",
            item["Dest"],
            1,
        )
    elif item["@sID"] in replaced_renames:
        item["Source"] = item["Dest"]
        item["Dest"] = re.sub(
            r"([^\x00-\x1F\x21-\x7F]+)(.*)",
            rf'\1{replaced_renames[item["@sID"]]} \2',
            item["Dest"],
            1,
        )
    elif item["@sID"] in plan_hunter_replaced_renames:
        item["Source"] = item["Dest"]
        item["Dest"] = re.sub(
            r"([^\x00-\x1F\x21-\x7F]+)(.*)",
            rf'\1{plan_hunter_replaced_renames[item["@sID"]]} \2',
            item["Dest"],
            1,
        )

    if item["@sID"] in clearer_descriptions_renames:
        item["Source"] = item["Dest"]
        if re.search(r"([^\x00-\x1F\x21-\x7F]+)(.*)([^\x00-\x1F\x20-\x7F]+)", item["Dest"]):
            item["Dest"] = re.sub(
                r"([^\x00-\x1F\x21-\x7F]+)(.*)([^\x00-\x1F\x20-\x7F]+)",
                rf"\1{clearer_descriptions_renames[item['@sID']]} \3",
                item["Dest"],
                1,
            )
        elif re.search(r"([^\x00-\x1F\x20-\x7F]+)(.*)", item["Dest"]):
            item["Dest"] = re.sub(
                r"([^\x00-\x1F\x21-\x7F]+)(.*)",
                rf"\1{clearer_descriptions_renames[item['@sID']]}",
                item["Dest"],
                1,
            )
        else:
            item["Dest"] = clearer_descriptions_renames[item["@sID"]]
log.info("Done!")

log.info("Writing output XML...")
output = xmltodict.unparse(icon_file, pretty=True)
with open("out.xml", "w", encoding="utf-8") as r:
    r.write(output)
log.info("Done!")