Calculate consumption rates and needed production buildings for Anno 1404.

anno4c2.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
##
# @file         ann4c2.py
# @author       redxef
# @since        2019-02-19
#
# @brief        Calculate consumption rates and needed production buildings for Anno 1404
#               This program has no hard-coded values and relies solely on game files of
#               Anno 1404, it has been written for IAAM and should be easy enough to adapt
#               to other versions and patches of the game.
#               To use this script do the following:
#               1. create a directory called 'res' in the same directory as this script.
#               2. Extract the patch0.rda into this folder using
#                  [RDAExplorer](https://github.com/lysannschlegel/RDAExplorer)
#                  path to patch0.rda: xxx\Ubisoft\Related Designs\IAAM 1404\addon\patch0.rda
#               This script basically needs the following 3 files:
#                - .\res\addon\addondata\config\game\assets.xml
#                - .\res\addon\addondata\config\game\properties.xml
#                - .\res\addon\data\loca\(ger|eng|fra)\txt\guids.txt
#               Every thing else can be deleted.

import sys
import os
import math
import collections
import codecs
import getopt
import xml.etree.ElementTree as xmlet


def get_patches_guids(res, lang='ger'):
    r = []
    guids_path = os.path.join('{res}', '{baseloc}', '{dmd}', 'loca', '{lang}', 'txt', 'guids.txt')
    for baseloc in ('maindata', 'addon'):
        try:
            for ent in os.listdir('{res}/{baseloc}'.format(res=res, baseloc=baseloc)):
                if baseloc == 'maindata':
                    dmd = os.path.join(ent, 'data')
                elif baseloc == 'addon':
                    dmd = ent
                file = guids_path.format(res=res, baseloc=baseloc, dmd=dmd, lang=lang)
                if os.path.isfile(file):
                    r += [file]
        except Exception as e:
            # print(e)
            pass
    return r

def get_patches_generic_file(res, filename):
    r = []
    assets_path = os.path.join('{res}', '{baseloc}', '{dmd}', 'config', 'game', '{filename}')
    for baseloc in ('maindata', 'addon'):
        try:
            for ent in os.listdir('{res}/{baseloc}'.format(res=res, baseloc=baseloc)):
                if baseloc == 'maindata':
                    dmd = os.path.join(ent, 'data')
                elif baseloc == 'addon':
                    dmd = ent
                file = assets_path.format(res=res, baseloc=baseloc, dmd=dmd, filename=filename)
                if os.path.isfile(file):
                    r += [file]
        except Exception as e:
            # print(e)
            pass
    return r

def get_patches_assets(res):
    return get_patches_generic_file(res, 'assets.xml')

def get_patches_game_properties(res):
    return get_patches_generic_file(res, 'properties.xml')

def load_guids(guids_txt):
    guids = {}
    f = open(guids_txt, 'rb')
    text = f.read()
    assert text.startswith(codecs.BOM_UTF16_LE)
    text = text[len(codecs.BOM_UTF16_LE):]
    text = text.decode('utf-16le')
    for line in text.splitlines():
        line = line.split('#', 1)[0]
        line = line.strip()
        try:
            num, text = line.split('=')
        except ValueError:
            continue
        num = int(num)
        guids[num] = text
    f.close()
    return guids

def load_xml(xml):
    tree = xmlet.parse(xml)
    return tree.getroot()

class base_object(object):

    def __repr__(self):
        s = '<{}('.format(type(self).__name__)
        for k, v in sorted(self.__dict__.items()):
            s += '{}={}, '.format(k, v)
        s = s[:-2]
        s += ')>'
        return s

class Product(base_object):

    def __init__(self, guid=None, name=None, loca_name=None):
        self.guid = guid
        self.name = name
        self.loca_name = loca_name

    def __lt__(self, other):
        return self.guid < other.guid

    def __gt__(self, other):
        return self.guid > other.guid

    def __hash__(self):
        return hash(self.guid)

    def __eq__(self, other):
        return (isinstance(other, type(self))
                and self.guid == other.guid
                and self.name == other.name
                and self.loca_name == other.loca_name)

class ProductFactory(base_object):

    def __init__(self, props, guids):
        self.props = props
        self.guids = guids
        self.products = {}
        self._balancing = None
        self._loaded = False

        for child in self.props.find('Groups').findall('Group'):
            if child.find('Name').text == 'Balancing':
                self._balancing = child
                break

    def _register_product(self, p):
        self.products[p.guid] = p
        self.products[p.name] = p
        self.products[p.loca_name] = p

    def _get_product_guid(self, guid):
        piguid = self._balancing.find('DefaultValues').find('GUIBalancing').find('ProductIconGUID')
        for c in piguid:
            if int(c.text) == guid:
                p_guid = guid
                p_name = c.tag
                p_loca_name = self.guids[p_guid]
                return Product(guid=p_guid, name=p_name, loca_name=p_loca_name)
        return None

    def _get_product_name(self, name):
        piguid = self._balancing.find('DefaultValues').find('GUIBalancing').find('ProductIconGUID')
        for c in piguid:
            if c.tag == name:
                p_guid = int(c.text)
                p_name = name
                p_loca_name = self.guids[p_guid]
                return Product(guid=p_guid, name=p_name, loca_name=p_loca_name)
        return None

    def _get_product_loca_name(self, loca_name):
        piguid = self._balancing.find('DefaultValues').find('GUIBalancing').find('ProductIconGUID')
        for c in piguid:
            if self.guids[int(c.text)] == loca_name:
                p_guid = int(c.text)
                p_name = c.tag
                p_loca_name = loca_name
                return Product(guid=p_guid, name=p_name, loca_name=p_loca_name)
        return None

    def load(self):
        if self._loaded:
            return
        piguid = self._balancing.find('DefaultValues').find('GUIBalancing').find('ProductIconGUID')
        for c in piguid:
            self.get_product(guid=int(c.text))
        self._loaded = True
        return

    def items(self):
        if not self._loaded:
            self.load()
        for k, v in self.products.items():
            if type(k) is int:
                yield v

    def get_product(self, guid=None, name=None, loca_name=None):
        for t, key in zip(('g', 'n', 'l'), (guid, name, loca_name)):
            if key is None:
                continue
            try:
                p = self.products[key]
            except KeyError:
                p = None
                if t == 'g' and p is None:
                    p = self._get_product_guid(key)
                if t == 'n' and p is None:
                    p = self._get_product_name(key)
                if t == 'l' and p is None:
                    p = self._get_product_loca_name(key)
                if p is not None:
                    self._register_product(p)
            return p

class ProductionBuilding(base_object):

    default_produce_time = 20000

    def __init__(self):
        self.guid = None
        self.name = None
        self.loca_name = None
        self.produces = None
        self.produce_time = self.default_produce_time
        self.produce_per_min = None
        self.req_product1_type = None
        self.req_product1_amount = None
        self.req_product2_type = None
        self.req_product2_amount = None

    def update(self):
        self.produce_per_min = int(60E6/self.produce_time)
        if self.req_product1_type is not None and self.req_product1_amount is None:
            self.req_product1_amount = 1000
        if self.req_product2_type is not None and self.req_product2_amount is None:
            self.req_product2_amount = 1000

class ResidenceBuilding(base_object):

    def __init__(self):
        self.guid = None
        self.name = None
        self.min_people = None
        self.max_people = None

def load_residence_buildings(assets):
        for child in assets.find('Groups').findall('Group'):
            if child.find('Name').text == 'Objects':
                objects = child.find('Groups')
        for child in list(objects):
            if child.find('Name').text == 'PlayerBuildings':
                playerbuildings = child.find('Groups')
        for child in list(playerbuildings):
            if child.find('Name').text == 'Residence':
                residence = child.find('Assets')


        res_buildings = {}

        for asset in residence.findall('Asset'):
            if asset.find('Template').text != 'ResidenceBuilding':
                continue
            field_guid = asset.find('Values').find('Standard').find('GUID')
            field_build_lvl = asset.find('Values').find('Building').find('BuildingLevel')
            field_min_residence_count = asset.find('Values').find('ResidenceBuilding').find('MinResidentCount')
            field_max_residence_count = asset.find('Values').find('ResidenceBuilding').find('MaxResidentCount')
            field_filename_rand = asset.find('Values').find('Object').find('Variations').find('Item').find('Filename')
            guid = int(field_guid.text)
            build_lvl = field_build_lvl.text

            try:
                min_people = int(field_min_residence_count.text)
            except AttributeError:
                min_people = 0

            try:
                max_people = int(field_max_residence_count.text)
            except AttributeError:
                max_people = 0

            if max_people == 0:
                continue
            if 'iaam' in field_filename_rand.text:
                continue

            build = ResidenceBuilding()
            build.guid = guid
            build.name = build_lvl
            build.min_people = min_people
            build.max_people = max_people
            res_buildings[build_lvl] = build
        return res_buildings

class ProductionBuildingFactory(base_object):

    def __init__(self, guids, product_factory, assets):
        self._guids = guids
        self._product_factory = product_factory
        self._assets = assets
        self.buildings = []

    def load(self):
        for child in self._assets.find('Groups').findall('Group'):
            if child.find('Name').text == 'Objects':
                objects = child.find('Groups')
        for child in list(objects):
            if child.find('Name').text == 'PlayerBuildings':
                playerbuildings = child.find('Groups')
        for child in list(playerbuildings):
            if child.find('Name').text == 'Production':
                production = child.find('Groups')
        for child in list(production):
            if child.find('Name').text == 'Farm':
                farm = child.find('Assets')
                other_farm = child.find('Groups')
            elif child.find('Name').text == 'Resource':
                resource = child.find('Assets')
            elif child.find('Name').text == 'Factory':
                factory = child.find('Assets')
        for child in other_farm.findall('Group'):
            if child.find('Name').text == 'Plantation':
                plant_farm = child.find('Assets')
            elif child.find('Name').text == 'Animalfarm':
                animal_farm = child.find('Groups')

        for child in list(farm) + list(plant_farm) + list(resource) + list(factory):
            field_guid = child.find('Values').find('Standard').find('GUID')
            field_name = child.find('Values').find('Standard').find('Name')
            if child.find('Values').find('WareProduction') is None:
                continue
            field_produces = child.find('Values').find('WareProduction').find('Product')
            field_produce_time = child.find('Values').find('WareProduction').find('ProductionTime')
            field_fac_rawmat1 = None
            field_fac_rawmat2 = None
            field_fac_rawneed1 = None
            field_fac_rawneed2 = None

            field_fac = child.find('Values').find('Factory')
            if field_fac is not None:
                field_fac_rawmat1 = field_fac.find('RawMaterial1')
                field_fac_rawneed1 = field_fac.find('RawNeeded1')
                field_fac_rawmat2 = field_fac.find('RawMaterial2')
                field_fac_rawneed2 = field_fac.find('RawNeeded2')

            _pb = ProductionBuilding()
            if field_guid is not None:
                _pb.guid = int(field_guid.text)
                try:
                    _pb.loca_name = self._guids[_pb.guid]
                except KeyError:
                    pass
            if field_name is not None:
                _pb.name = field_name.text
            if field_produces is not None:
                _pb.produces = self._product_factory.get_product(name=field_produces.text)
            if field_produce_time is not None:
                _pb.produce_time = int(field_produce_time.text)

            if field_fac_rawmat1 is not None:
                _pb.req_product1_type = self._product_factory.get_product(name=field_fac_rawmat1.text)
            if field_fac_rawneed1 is not None:
                _pb.req_product1_amount = int(field_fac_rawneed1.text)
            if field_fac_rawmat2 is not None:
                _pb.req_product2_type = self._product_factory.get_product(name=field_fac_rawmat2.text)
            if field_fac_rawneed2 is not None:
                _pb.req_product2_amount = int(field_fac_rawneed2.text)

            _pb.update()
            self.buildings += [_pb]
        return self

    def items(self):
        for b in self.buildings:
            yield b

    def get_building_by_product(self, product):
        for building in self.buildings:
            if building.produces == product:
                yield building
        yield None

    def get_building_requirements(self, building):
        for pre_building0, pre_building1 in zip(
                        self.get_building_by_product(building.req_product1_type),
                        self.get_building_by_product(building.req_product2_type)):
            building_0 = None, None
            building_1 = None, None

            if building.req_product1_type is not None and pre_building0 is not None:
                cnt_buildings = (1/1000)*(building.produce_per_min * building.req_product1_amount) / pre_building0.produce_per_min
                building_0 = pre_building0, cnt_buildings
            if building.req_product2_type is not None and pre_building1 is not None:
                cnt_buildings = (1/1000)*(building.produce_per_min * building.req_product2_amount) / pre_building1.produce_per_min
                building_1 = pre_building1, cnt_buildings
            yield building_0, building_1

class InhabitantClass(base_object):

    def __init__(self, name=None):
        self.name = name
        self.wants = []

    def add_want(self, w):
        self.wants += [w]
        return self

def get_demands(product_factory, props):
    pf = product_factory
    demands = {}
    for child in props.find('Groups').findall('Group'):
        if child.find('Name').text == 'Balancing':
            balancing = child
    demand_amount = balancing.find('DefaultValues').find('Balancing').find('DemandAmount')

    nomad = InhabitantClass('Nomad')
    ambassador = InhabitantClass('Ambassador')
    peasant = InhabitantClass('Peasant')
    citizen = InhabitantClass('Citizen')
    patrician = InhabitantClass('Patrician')
    nobleman = InhabitantClass('Nobleman')
    for child in demand_amount.find(nomad.name):
        nomad.add_want((pf.get_product(name=child.tag), int(child.text)))
    for child in demand_amount.find(ambassador.name):
        ambassador.add_want((pf.get_product(name=child.tag), int(child.text)))
    for child in demand_amount.find(peasant.name):
        peasant.add_want((pf.get_product(name=child.tag), int(child.text)))
    for child in demand_amount.find(citizen.name):
        citizen.add_want((pf.get_product(name=child.tag), int(child.text)))
    for child in demand_amount.find(patrician.name):
        patrician.add_want((pf.get_product(name=child.tag), int(child.text)))
    for child in demand_amount.find(nobleman.name):
        nobleman.add_want((pf.get_product(name=child.tag), int(child.text)))
    return (nomad, ambassador, peasant, citizen, patrician, nobleman)

def cmd_calc(product_factory, production_building_factory, demands, res_builds, skip_unneeded, interactive):
    pf = product_factory
    pbf = production_building_factory
    cnt_inhab = {
        'Nomad': 0,
        'Ambassador': 0,
        'Peasant': 0,
        'Citizen': 0,
        'Patrician': 0,
        'Nobleman': 0
    }

    for k in ('Peasant', 'Citizen', 'Patrician', 'Nobleman', 'Nomad', 'Ambassador'):
        try:
            inhabs = input('{}: '.format(k)).lower()
            if inhabs.endswith('h'):
                res_build = res_builds[k]
                cnt_inhab[k] = int(eval(inhabs[:-1])) * res_build.max_people
            else:
                cnt_inhab[k] = int(eval(inhabs))
        except (ValueError, SyntaxError):
            cnt_inhab[k] = 0

    product_demand_sum = {}

    for inhab_class in demands:
        for product, count in inhab_class.wants:
            kgpmin = count * cnt_inhab[inhab_class.name] * (1/100)
            try:
                pre_val = product_demand_sum[product]
            except KeyError:
                pre_val = 0
            product_demand_sum[product] = pre_val + kgpmin

    for k, v in product_demand_sum.items():
        product_demand_sum[k] = int(math.ceil(v))

    def print_indent_production_chain(building, building_cnt, indent):
        print('{}{:.2f} {}'.format(' '*indent, building_cnt, building.loca_name))
        for (b0, b0cnt), (b1, b1cnt) in pbf.get_building_requirements(building):
            if b0 is not None:
                print_indent_production_chain(b0, b0cnt * building_cnt, indent+2)
            if b1 is not None:
                print_indent_production_chain(b1, b1cnt * building_cnt, indent+2)
            if b0 is None and b1 is None:
                break

    for k, v in product_demand_sum.items():
        if skip_unneeded and v == 0:
            continue
        print('{}: {:.2f} t/min'.format(k.loca_name, v/1000))
        for building in pbf.get_building_by_product(k):
            if building is None:
                break
            building_cnt = v / building.produce_per_min
            print_indent_production_chain(building, building_cnt, 2)

def cmd_chain(product_factory, production_building_factory, interactive):
    pf = product_factory
    pbf = production_building_factory

    ware = input('enter ware: ').strip()
    prod = pf.get_product(name=ware)
    if prod == None:
        prod = pf.get_product(loca_name=ware)
    if prod == None:
        try:
            prod = pf.get_product(guid=int(ware))
        except ValueError:
            print('no such ware found')
            return

    def print_indent_production_chain(building, cnt, indent):
        print('{}{:.2f} {}, {}t/min'.format(' '*indent, cnt, building.loca_name, building.produce_per_min))
        for (b0, b0cnt), (b1, b1cnt) in pbf.get_building_requirements(building):
            if b0 is not None:
                print_indent_production_chain(b0, b0cnt * cnt, indent+2)
            if b1 is not None:
                print_indent_production_chain(b1, b1cnt * cnt, indent+2)
            if b0 is None and b1 is None:
                break

    prod_buildings = pbf.get_building_by_product(prod)
    for pb in prod_buildings:
        if pb is None:
            continue
        print_indent_production_chain(pb, 1, 0)

def cmd_list_products(product_factory, production_building_factory):
    pf = product_factory
    pbf = production_building_factory

    for prod in pf.items():
        print('{}: (guid: {}, name: {})'.format(prod.loca_name, prod.guid, prod.name))

_help_text = """NAME
    python anno4c2.py - Calculate consumtion rates of Inhabitants in
                        Anno 1404 (Venice) and IAAM 1404.

SYNOPSIS
    python anno4c2.py [-s] [--skip-unneeded] [-S] [--no-skip-unneeded]
    [-i] [--interactive] [-h] [--help] cmd

DESCRIPTION
    anno4c2.py has multiple commands (cmd) either the prominent ones are
    calc or chain.
    available commands:
     - calc
     - chain
     - list-products

    Command Description: `chain'
        When using chain, then the program will output the production chain
        of the given Product. The product can generally be specified with
         a: the localized name
         b: the guid
         c: the internal unpretty name for the product
        The output is the complete production chain with the raw resources
        indented the most.
        If there are two production facilities which produce the same
        resource on the same (indentation) level, then they represent
        alternatives (think Coal Mine and Charcoalburners Hut).
        The Number in front of the facility denotes how productive this
        Building has to be.
        Example: `Tools'
        Will output the following:
        ```
        1.00 Schmied, 2000t/min
          0.50 Erzschmelze, 2000t/min
            0.50 Eisenmine, 2000t/min
            0.50 Köhlerei, 2000t/min
            0.25 Tiefe Eisenmine, 4000t/min
            0.42 Kohlemine, 2400t/min
            0.25 Tiefe Kohlemine, 4000t/min
        ```
    Command Description: `calc'
        Calc will ask for the number of Inhabitants by class and produce
        a list of production facilities quiet similar to the output of
        chain.
    Command Description: `list-products'
        This will list all Products which can be demanded by Inhabitants.
        Each line contains one product. The format is as follows:
        <localized name>: guid: <guid of the product>, name: <internal name>
        Any of those thre values can be used to specify a ware when asked to
        do so in another command.

OPTIONS
    -s, --skip-unneeded:
        This option only applies when using the calc command.
        If a ware is not consumed by any class then dont list
        this product (Think early game with only citizens, it
        would be useless to show that 0 Breweries are needed).
        This is the default behaviour.
    -S, --no-skip-unneeded:
        The inverse of -s or --skip-unneeded.
    -i, --interactive:
        !!! Currently not implemented, input is only interactively possible
        Should the input to the commands happen interactive or
        be given via command line (when this flag is passed), the
        input happens interactively (duh).
        This is the default behaviour.
    -I, --no-interactive:
        !!! Currently not implemented, input is only interactively possible
        The opposite of -i or --interactive.
        If this is passed, then every input is passed with a argument.
"""

def main(argc, argv):
    print(argc)
    print(argv)

    # Options
    skip_unneeded = True
    interactive = True

    shopts = 'sShiI'
    loopts = ['skip-unneeded', 'no-skip-unneeded',
            'help', 'interactive', 'no-interactive']
    if len(argv) == 1:
        argv += ['--help']
        argc += 1

    opts, args = getopt.getopt(argv[1:], shopts, loopts)
    for o, a in opts:
        if o in ('-s', '--skip-unneeded'):
            skip_unneeded = True
        elif o in ('-S', '--no-skip-unneeded'):
            skip_unneeded = False
        elif o in ('-h', '--help'):
            print(_help_text)
            return
        elif o in ('-i', '--interactive'):
            interactive = True
        elif o in ('-I', '--no-interactive'):
            interactive = False

    if len(args) == 0:
        args = ['calc']

    guids_files = get_patches_guids('./res')
    asset_files = get_patches_assets('./res')
    properties_files = get_patches_game_properties('./res')
    guids = load_guids(guids_files[-1])
    assets = load_xml(asset_files[-1])
    props = load_xml(properties_files[-1])
    pf = ProductFactory(props, guids)
    pbf = ProductionBuildingFactory(guids, pf, assets).load()
    demands = get_demands(pf, props)
    res_builds = load_residence_buildings(assets)

    for arg in args:
        if arg == 'calc':
            cmd_calc(pf, pbf, demands, res_builds, skip_unneeded, interactive)
        elif arg == 'chain':
            cmd_chain(pf, pbf, interactive)
        elif arg == 'list-products':
            cmd_list_products(pf, pbf)


if __name__ == '__main__':
    main(len(sys.argv), sys.argv)

open_cmd.bat

%~d1
cd "%~p1"
call cmd

I changed the language in the script to 'eng' in line #32. The script returned an error after running on extract from RDAExplorerGUI:
Traceback (most recent call last):
File "anno4c2.py", line 652, in <module>
main(len(sys.argv), sys.argv)
File "anno4c2.py", line 634, in main
guids = load_guids(guids_files[-1])
IndexError: list index out of range

In my case the english guids.txt file was in the following path:
.\res\data\loca\eng
The script was looking for the file in:
.\res\addon\data\loca\eng\txt

I added another level of folders (\addon\) because RDAExplorer didn't create it.
I copied guids.txt to correct folder. No errors after that.

Thank you for the script!!