Generate DSP data.json for factorio-lab using extracted by uTinyRipper assets.

GenerateDataDSP.py

#!/usr/bin/python

import os
import sys
import yaml
import json
import pprint
import textwrap
import unityparser
from skimage import io

BASE_PATH = os.path.abspath(os.path.dirname(sys.argv[0]))
RESOURCES_PATH = os.path.join(BASE_PATH, "DSPGAME", "Assets", "Resources")
PROTOTYPES_PATH = os.path.join(RESOURCES_PATH, "prototypes")
TEXTURES_PATH = os.path.join(RESOURCES_PATH, "ui", "textures")

# This command will produce icons.png
'''
magick montage ^
  DSPGAME/Assets/Resources/ui/textures/sprites/icons/component-icon.png ^
  DSPGAME/Assets/Resources/ui/textures/sprites/icons/factory-icon.png ^
  DSPGAME/Assets/Resources/icons/tech/1606.png ^
  DSPGAME/Assets/Resources/icons/itemrecipe/*.png ^
  DSPGAME/Assets/Resources/icons/vein/*.png ^
  -geometry 64x64+0+0 -background transparent ^
  png32:icons.png
'''

NAMES = {}
IDS = {}
GRID = {}
ITEMS = {}
RECIPES = {}
PREFABS = {}
VEINS = {}
ICONS = {}

WATER_PUMP_ITEMS = [
    "water",
    "sulphuric-acid",
]

RECIPE_TYPES = {
    0:  "None",
    1:  "Smelt",
    2:  "Chemical",
    3:  "Refine",
    4:  "Assemble",
    5:  "Particle",
    6:  "Exchange",
    7:  "PhotonStore",
    8:  "Fractionate",
    15: "Research",
}

MINER_TYPES = {
    0:  "None",
    1:  "Water",
    2:  "Vein",
    3:  "Oil",
}

FUEL_TYPES = {
    0: 'None',
    1: 'Chemical',
    2: 'Nuclear',
    4: "Antimatter",
    8: 'Accumulator',
}

PRODUCERS = {
    'Smelt':        ["smelter"],
    'Chemical':     ["chemical-plant"],
    'Refine':       ["oil-refinery"],
    'Assemble':     ["assembler-1", "assembler-2", "assembler-3"],
    'Particle':     ["hadron-collider"],
    'Fractionate':  ["fractionator"],
    'Research':     ["lab"],
    'Water':        ["water-pump"],
    'Vein':         ["mining-drill"],
    'Oil':          ["oil-extractor"],
}


def GetName(Name):
    return NAMES[Name] if Name in NAMES else Name


def ToItemId(Path):
    return Path.split("/")[-1].lower()


def ToIntArray(Input):
    def Convert(x): return int.from_bytes(bytes.fromhex(x), byteorder='little')
    return [Convert(x) for x in textwrap.wrap(str(Input), 8)]


def ListIcons(IconsPath):
    Icons = {}
    for Filename in os.listdir(IconsPath):
        if Filename.endswith(".png"):
            IconId = Filename[:-4].lower()
            Icons[IconId] = os.path.join(IconsPath, Filename)
    return Icons


def ParseBeltDesc(Desc):
    return {
        'beltSpeed': Desc['speed']
    }


def ParseAssemblerDesc(Desc):
    return {
        'assemblerType': Desc['recipeType'],
        'assemblerSpeed': Desc['speedf'],
    }


def ParseLabDesc(Desc):
    return {
        'assembleSpeed': Desc['assembleSpeed'],
        'researchSpeed': Desc['researchSpeed'],
    }


def ParseMinerDesc(Desc):
    return {
        'minerType': Desc['minerType'],
        'miningSpeed': Desc['periodf'],
    }


def ParseFractionateDesc(Desc):
    return {
        'fractionateType': Desc['recipeType'],
        'needMaxCount': Desc['needMaxCount'],
        'productMaxCount': Desc['productMaxCount'],
        'oriProductMaxCount': Desc['oriProductMaxCount'],
    }


def ParsePowerDesc(Desc):
    return {
        'fuelMask': Desc['fuelMask'],
        'productId': Desc['productId'],
        'workEnergyPerTick': Desc['workEnergyPerTick'],
        'idleEnergyPerTick': Desc['idleEnergyPerTick'],
    }


def ParseInserterDesc(Desc):
    return {
        'inserterCanStack': Desc['canStack'],
        'inserterStackSize': Desc['stackSize'],
    }


def ParseStationDesc(Desc):
    if Desc['isCollector']:
        return {
            'stationCollectSpeed': Desc['collectSpeed']
        }
    return {}


SCRIPT_TYPES = {
    'f4df938feead02b4a9ac37389ccb926d': ParseBeltDesc,
    '0e3d842d96462e9459f1989fd341389e': ParseAssemblerDesc,
    'b8894acb258f3c749bd3046541329178': ParsePowerDesc,
    '89e018bb65efaf848958841d52240534': ParseLabDesc,
    '8dce58da3c359c14da23e9f8106cdca2': ParseMinerDesc,
    '1f03319d47b5d2a4798e64a8cf2a994c': ParseFractionateDesc,
    'a883c8cc7e279dc44b151589ae9f5c5d': ParseInserterDesc,
    '8fef6d3ce26c11d48ba2d9b5fe033a3e': ParseStationDesc,
}

PREFAB_NAME_PROXY = {
    'assembler-mk-1': "assembler-1",
    'assembler-mk-2': "assembler-2",
    'assembler-mk-3': "assembler-3",
    'power-generator': "fuel-plant",
}

UI_ICONS_PATH = os.path.join(TEXTURES_PATH, "sprites", "icons")
ICNOS_PATH = os.path.join(RESOURCES_PATH, "icons")

ICON_PATHS = {
    'components': os.path.join(UI_ICONS_PATH, "component-icon.png"),
    'buildings': os.path.join(UI_ICONS_PATH, "factory-icon.png"),
    'gas-mining-tech': os.path.join(ICNOS_PATH, "tech", "1606.png"),
}
ICON_PATHS.update(ListIcons(os.path.join(ICNOS_PATH, "itemrecipe")))
ICON_PATHS.update(ListIcons(os.path.join(ICNOS_PATH, "vein")))

ICON_INDEX = 0
for IconId, IconPath in ICON_PATHS.items():
    Image = io.imread(IconPath)[:, :, :-1]
    Average = Image.mean(axis=0).mean(axis=0)

    Icon = {
        "color": "#%02X%02X%02X" % (int(Average[0]), int(Average[1]), int(Average[2])),
        "id": IconId,
        "position": "%dpx %dpx" % (-64 * int(ICON_INDEX % 13), -64 * int(ICON_INDEX / 13))
    }

    ICONS[IconId] = Icon
    ICON_INDEX = ICON_INDEX + 1

for Item in WATER_PUMP_ITEMS:
    VeinId = Item + "-vein"
    ICONS[VeinId] = {**ICONS[Item], 'id': VeinId}

with open(os.path.join(PROTOTYPES_PATH, "StringProtoSet.asset"), "r", encoding='utf8') as stream:
    try:
        Document = yaml.load(stream, Loader=unityparser.loader.UnityLoader)
        for Entry in Document.dataArray:
            NAMES[Entry['Name']] = Entry['ENUS']
    except yaml.YAMLError as e:
        print(e)

with open(os.path.join(PROTOTYPES_PATH, "ItemProtoSet.asset"), "r", encoding='utf8') as stream:
    try:
        Document = yaml.load(stream, Loader=unityparser.loader.UnityLoader)
        for Entry in Document.dataArray:
            Name = GetName(Entry['Name'])
            ItemId = ToItemId(Entry['IconPath'])
            IsBuilding = bool(Entry['IsEntity']) and bool(Entry['CanBuild'])
            Item = {
                'category': "buildings" if IsBuilding else "components",
                'id': ItemId,
                'name': Name,
                'row': int((Entry['GridIndex'] % 1000) / 100 - 1),
                'stack': int(Entry['StackSize']),
            }

            if (Entry['FuelType'] > 0):
                Item['fuel'] = {
                    'category': FUEL_TYPES[Entry['FuelType']].lower(),
                    'value': float(Entry['HeatValue']) / 1000000.0,
                }

            IDS[Entry['ID']] = ItemId
            GRID[Entry['GridIndex']] = ItemId
            ITEMS[ItemId] = Item
    except yaml.YAMLError as e:
        print(e)

with open(os.path.join(PROTOTYPES_PATH, "RecipeProtoSet.asset"), "r", encoding='utf8') as stream:
    try:
        Document = yaml.load(stream, Loader=unityparser.loader.UnityLoader)
        for Entry in Document.dataArray:
            Name = GetName(Entry['Name'])

            Items = [IDS[x] for x in ToIntArray(Entry['Items'])]
            ItemCounts = ToIntArray(Entry['ItemCounts'])

            Results = [IDS[x] for x in ToIntArray(Entry['Results'])]
            ResultCounts = ToIntArray(Entry['ResultCounts'])

            if bool(Entry['Explicit']):
                RecipeId = ToItemId(Entry['IconPath'])
            else:
                RecipeId = next(iter(Results))

            Time = float(Entry['TimeSpend']) / 60.0

            if RECIPE_TYPES[Entry['Type']] == "Fractionate":
                Time = 1.0
                Fraction = float(ResultCounts[0]) / float(ItemCounts[0])
                ItemCounts[0] = 1.0
                ResultCounts[0] = Fraction
                Results.append(Items[0])
                ResultCounts.append(1.0 - Fraction)

            RECIPES[RecipeId] = {
                'id': RecipeId,
                'name': Name,
                'in': dict(zip(Items, ItemCounts)),
                'out': dict(zip(Results, ResultCounts)),
                'time': Time,
                'producers': PRODUCERS[RECIPE_TYPES[Entry['Type']]],
            }
    except yaml.YAMLError as e:
        print(e)

with open(os.path.join(PROTOTYPES_PATH, "VeinProtoSet.asset"), "r", encoding='utf8') as stream:
    try:
        Document = yaml.load(stream, Loader=unityparser.loader.UnityLoader)
        for Entry in Document.dataArray:
            VeinId = ToItemId(Entry['IconPath'])
            IsVein = bool(Entry['MinerBaseModelIndex'])
            Vein = {
                'id': VeinId,
                'type': "Vein" if IsVein else "Oil",
                'out': {IDS[Entry['MiningItem']]: 1},
            }
            VEINS[VeinId] = Vein
    except yaml.YAMLError as e:
        print(e)

for Item in WATER_PUMP_ITEMS:
    VeinId = Item + "-vein"
    VEINS[VeinId] = {
        'id': VeinId,
        'type': "Water",
        'out': {Item: 1},
    }

PREFAB_PATH = os.path.join(RESOURCES_PATH, "entities", "prefabs")
for FileName in os.listdir(PREFAB_PATH):
    if not FileName.endswith(".prefab"):
        continue

    Name = FileName[:-7]
    PrefabId = PREFAB_NAME_PROXY[Name] if Name in PREFAB_NAME_PROXY else Name
    Prefab = {}

    with open(os.path.join(PREFAB_PATH, FileName), "r", encoding='utf8') as stream:
        try:
            Docs = yaml.load_all(stream, Loader=unityparser.loader.UnityLoader)
            for Entry in [Entry.get_serialized_properties_dict() for Entry in Docs]:
                if 'm_Script' in Entry:
                    Guid = Entry['m_Script']['guid']
                    if Guid in SCRIPT_TYPES:
                        Prefab.update(SCRIPT_TYPES[Guid](Entry))
        except yaml.YAMLError as e:
            print(e)

    PREFABS[PrefabId] = Prefab

for ItemId, Prefab in PREFABS.items():
    if not ItemId in ITEMS:
        continue

    if 'beltSpeed' in Prefab:
        ITEMS[ItemId]['belt'] = {
            'speed': float(Prefab['beltSpeed']) * 6.0
        }
        ITEMS[ItemId]['module'] = {
            'consumption': 0.0,
            'speed': float(Prefab['beltSpeed']) - 1.0
        }

    Factory = {}
    if 'assemblerSpeed' in Prefab:
        Factory['speed'] = float(Prefab['assemblerSpeed'])
    if 'workEnergyPerTick' in Prefab and Prefab['workEnergyPerTick'] > 0:
        Factory['type'] = "electric"
        Factory['usage'] = float(Prefab['workEnergyPerTick']) * 0.06
    if 'idleEnergyPerTick' in Prefab and Prefab['idleEnergyPerTick'] > 0:
        Factory['type'] = "electric"
        Factory['drain'] = float(Prefab['idleEnergyPerTick']) * 0.06
    if 'fuelMask' in Prefab and Prefab['fuelMask'] > 0:
        Factory['type'] = "burner"
        Factory['category'] = FUEL_TYPES[Prefab['fuelMask']].lower()
    if 'minerType' in Prefab:
        Factory['mining'] = True
        Factory['speed'] = 1.0
    if 'assembleSpeed' in Prefab:
        Factory['speed'] = float(Prefab['assembleSpeed'])
    if 'productId' in Prefab and Prefab['productId'] > 0:
        Factory['speed'] = 1.0
    if 'fractionateType' in Prefab:
        Factory['modules'] = 1
        Factory['speed'] = 6.0  # belt speed multiplier
    if 'stationCollectSpeed' in Prefab:
        Factory['mining'] = True
        Factory['speed'] = float(Prefab['stationCollectSpeed'])

    if 'minerType' in Prefab and Prefab['minerType'] > 0:
        del Factory['type']
        del Factory['usage']
        del Factory['drain']

    if len(Factory) > 0:
        Factory['speed'] = Factory['speed'] if 'speed' in Factory else 1.0
        ITEMS[ItemId]['factory'] = Factory

    if 'productId' in Prefab and Prefab['productId'] > 0:
        RecipeId = IDS[Prefab['productId']]
        RECIPES[RecipeId] = {
            'id': RecipeId,
            'mining': True,
            'out': {RecipeId: 1},
            'time': 1.0,
            'producers': [ItemId],
        }

    if 'minerType' in Prefab and Prefab['minerType'] > 0:
        MinerType = MINER_TYPES[Prefab['minerType']]
        for VeinId, Vein in VEINS.items():
            if Vein['type'] != MinerType:
                continue

            RECIPES[VeinId] = {
                'id': VeinId,
                'mining': True,
                'time': float(Prefab['miningSpeed']),
                'out': Vein['out'],
                'producers': PRODUCERS[Vein['type']],
            }

with open(os.path.join(PROTOTYPES_PATH, "ThemeProtoSet.asset"), "r", encoding='utf8') as stream:
    try:
        Document = yaml.load(stream, Loader=unityparser.loader.UnityLoader)
        for Entry in Document.dataArray:
            if Entry['PlanetType'] != 5:
                continue

            PlanetType = 'gas' if Entry['Temperature'] > 0 else 'ice'
            RecipeId = PlanetType + "-giant"
            if RecipeId in RECIPES:
                continue

            GasItems = [IDS[x] for x in ToIntArray(Entry['GasItems'])]
            GasSpeeds = [float(x) for x in Entry['GasSpeeds']]

            TotalHeat = sum([Speed * ITEMS[ItemId]['fuel']['value']
                             for ItemId, Speed in zip(GasItems, GasSpeeds)])

            Producer = ITEMS['orbital-collector']

            WorkEnergy = float(Producer['factory']['usage']) / 1000.0
            CollectorSpeed = float(Producer['factory']['speed'])
            Coefficient = 1.0 - WorkEnergy / (CollectorSpeed * TotalHeat)
            GasCounts = [round(Speed * CollectorSpeed * Coefficient, 2)
                         for Speed in GasSpeeds]

            RECIPES[RecipeId] = {
                'id': RecipeId,
                'name': GetName(Entry['DisplayName']),
                'mining':  True,
                'time': CollectorSpeed,
                'out': dict(zip(GasItems, GasCounts)),
                'producers': [Producer['id']],
            }

            ICONS[RecipeId] = {**ICONS['gas-mining-tech'], 'id': RecipeId}

    except yaml.YAMLError as e:
        print(e)


DATA = {
    'categories': [
        {'id': "components", 'name': "Components"},
        {'id': "buildings", 'name': "Buildings"},
    ],
    'icons': list(ICONS.values()),
    'items': [ITEMS[ItemId] for _, ItemId in sorted(GRID.items())],
    'recipes': list(RECIPES.values()),
    'defaults': {
        'modIds': [],
        'minBelt': "belt-1",
        'maxBelt': "belt-3",
        'fuel': "coal-ore",
        'disabledRecipes': [],
        'minFactoryRank': ["assembler-1"],
        'maxFactoryRank': ["assembler-3"],
        'moduleRank': []
    },
    'limitations': {'productivity-module': []}
}

print(json.dumps(DATA))

requirements.txt

scikit-image>=0.18.1
PyYAML>=5.1
unityparser>=1.0.0

Hey, i tried to use this script but it throws me an error just when i use .load from yaml library, do you have any idea why?
Exception has occurred: AttributeError
'NoneType' object has no attribute 'set'
Document = yaml.load(stream, Loader=unityparser.loader.UnityLoader)
Looks like it doesn't recognize the file but i'm not sure why

Hey, i tried to use this script but it throws me an error just when i use .load from yaml library, do you have any idea why? Exception has occurred: AttributeError 'NoneType' object has no attribute 'set' Document = yaml.load(stream, Loader=unityparser.loader.UnityLoader) Looks like it doesn't recognize the file but i'm not sure why

Hi, Unity YAML is not a strict YAML as I remember, you need to fix source code of YAML parser or fix Unity files. I don't know what I did myself exactly, maybe preprocessed Unity files first to edit invalid lines. I really don't remember.

Hey, i tried to use this script but it throws me an error just when i use .load from yaml library, do you have any idea why? Exception has occurred: AttributeError 'NoneType' object has no attribute 'set' Document = yaml.load(stream, Loader=unityparser.loader.UnityLoader) Looks like it doesn't recognize the file but i'm not sure why

Hi, Unity YAML is not a strict YAML as I remember, you need to fix source code of YAML parser or fix Unity files. I don't know what I did myself exactly, maybe preprocessed Unity files first to edit invalid lines. I really don't remember.

Ty for the hint, i fixed it