Parser for LionHead's Black & White CTR files containing animations, hair and extra data for the hand and creatures (.hbn, cbn)

ctr_parser.py

import ctypes
import itertools
from collections import OrderedDict
import pprint
from pathlib import Path


class Vec3(ctypes.Structure):
    _fields_ = [
        ("x", ctypes.c_float),
        ("y", ctypes.c_float),
        ("z", ctypes.c_float),
    ]

    def __repr__(self):
        return "vec3{%f, %f, %f}" % (self.x, self.y, self.z)


def parse_spec_file(section_name: str, spec_version: int):
    spec_files = {
        'Hand': "Data/hndspec%d.txt",
        'Creature': "Data/ctrspec%d.txt",
    }

    lines = [i.rstrip() for i in open(spec_files[section_name] % spec_version).readlines()]
    # First line is spec version
    assert(int(lines[0]) == spec_version)
    lines.pop(0)
    # File is ended with E
    assert(lines[-1] == "E")
    lines.pop(-1)

    animations = OrderedDict()
    current_key = None
    total_animations = 0
    for line in lines:
        if line[0] == '=':
            current_key = line[1:]
            animations[current_key] = []
            continue
        assert(current_key is not None)
        animations[current_key].append((line[1:], line[0]))
        total_animations += 1

    # Sanity checks
    if section_name == 'Hand':
        assert(total_animations == 69)
        assert(spec_version == 5)
    elif section_name == 'Creature':
        assert (total_animations == 232)
        assert (spec_version == 27)
    else:
        assert False

    return animations


def parse_animation(file, section_offset, animation_names, anim_offsets, variant: str, verbose=False):
    class AnimationHeader(ctypes.Structure):
        _fields_ = [
            ("field_0x0", ctypes.c_uint32),  # TODO: unknown, possibly duration or offset
            ("field_0x4", ctypes.c_uint32),  # TODO: unknown, either 0 or 1. Seem to be 1 when type C and 0 when not
            ("field_0x8", ctypes.c_float * 5),  # TODO: unknown
            ("frame_count", ctypes.c_uint32),
            ("mesh_bone_count", ctypes.c_uint32),
            ("rotated_joint_count", ctypes.c_uint32),
            ("translated_joint_count", ctypes.c_uint32),
        ]

        def __repr__(self):
            return (
                f"{self.__class__.__name__}(\n"
                f"    field_0x0=0x{self.field_0x0:x},\n"
                f"    field_0x4=0x{self.field_0x4:x},\n"
                f"    field_0x8={list(self.field_0x8)},\n"
                f"    frame_count={self.frame_count},\n"
                f"    mesh_bone_count={self.mesh_bone_count},\n"
                f"    rotated_joint_count={self.rotated_joint_count},\n"
                f"    translated_joint_count={self.translated_joint_count},\n"
                f")"
            )

    bytes_read = 0

    if verbose:
        print(f"animset of {variant}")

    # Follow the offsets in anim set to parse the animations
    for name, offset in zip(animation_names, anim_offsets):
        if offset == 0:
            continue

        file.seek(section_offset + offset)

        # Animation starts with a header
        header = AnimationHeader()
        bytes_read += file.readinto(header)

        if verbose:
            print(f"header(\"{name}\")={header}")

        # Then an array of indices of size header.frame_field_0x0_count
        rotated_bone_indices = (ctypes.c_uint32 * header.rotated_joint_count)()
        bytes_read += file.readinto(rotated_bone_indices)

        if verbose:
            print(f"rotated_bone_indices={list(rotated_bone_indices)}")

        # Then an array of indices of size header.frame_field_0x4_count
        translated_bone_indices = (ctypes.c_uint32 * header.translated_joint_count)()
        bytes_read += file.readinto(translated_bone_indices)
        if verbose:
            print(f"translated_bone_indices={list(translated_bone_indices)}")

        class AnimationFrame(ctypes.Structure):
            _fields_ = [
                ("rotation_keyframes", Vec3 * header.rotated_joint_count),
                ("translation_keyframes", Vec3 * header.translated_joint_count),
            ]

            def __repr__(self):
                return (
                    f"{self.__class__.__name__}(\n"
                    f"    rotation_keyframes={[i for i in self.rotation_keyframes]},\n"
                    f"    translation_keyframes={[i for i in self.translation_keyframes]},\n"
                    f")"
                )

        # Finally, there is an array of size header.frame_count
        frames = (AnimationFrame * header.frame_count)()
        bytes_read += file.readinto(frames)
        if verbose:
            print(f"frames={pprint.pformat([i for i in frames])}")

    return bytes_read


def parse_hair_group(file, section_offset):
    class HairGroupHeaderMember(ctypes.Structure):
        """Function unknown"""
        _fields_ = [
            ("field_0x0", ctypes.c_uint32),
            ("field_0x4", ctypes.c_uint32),
            ("field_0x8", ctypes.c_uint32),
            ("field_0xc", ctypes.c_uint32),
            ("field_0x10", ctypes.c_uint32),
            ("field_0x14", ctypes.c_uint32),
            ("field_0x18", ctypes.c_uint32),
        ]

        def __repr__(self):
            return (
                f"{self.__class__.__name__}(\n"
                f"    field_0x0=0x{self.field_0x0:x},\n"
                f"    field_0x4=0x{self.field_0x4:x},\n"
                f"    field_0x8=0x{self.field_0x8:x},\n"
                f"    field_0xc=0x{self.field_0xc:x},\n"
                f"    field_0x10=0x{self.field_0x10:x},\n"
                f"    field_0x14=0x{self.field_0x14:x},\n"
                f"    field_0x18=0x{self.field_0x18:x},\n"
                f")"
            )

    class HairGroupHeader(ctypes.Structure):
        _fields_ = [
            ("field_0x0", ctypes.c_uint32),
            ("hair_count", ctypes.c_uint32),
            ("count_0x8", ctypes.c_uint32),
            ("field_0xc", ctypes.c_uint32),
            ("field_0x10", HairGroupHeaderMember * 3),
        ]

        def __repr__(self):
            return (
                f"{self.__class__.__name__}(\n"
                f"    field_0x0=0x{self.field_0x0:x},\n"
                f"    hair_count=0x{self.hair_count:x},\n"
                f"    count_0x8=0x{self.count_0x8:x},\n"
                f"    field_0xc=0x{self.field_0xc:x},\n"
                f"    field_0x10={pprint.pformat([i for i in self.field_0x10])},\n"
                f")"
            )

    bytes_read = 0

    hair_group_header = HairGroupHeader()
    bytes_read += file.readinto(hair_group_header)
    print(f"hair_group_header={hair_group_header}")

    # Now parse each hair which has an intersection structure
    class HairIntersection(ctypes.Structure):
        _fields_ = [
            ("field_0x0", ctypes.c_uint32),
            ("field_0x4", ctypes.c_uint32),
            ("field_0x8", ctypes.c_uint32),
            ("field_0xc", ctypes.c_uint32),
            ("field_0x10", ctypes.c_uint32),
            ("field_0x14", ctypes.c_uint32),
            ("field_0x18", ctypes.c_uint32),
            ("field_0x1c", ctypes.c_uint32),
            ("field_0x20", ctypes.c_uint32),
        ]

        def __repr__(self):
            return (
                f"{self.__class__.__name__}(\n"
                f"    field_0x0=0x{self.field_0x0:x},\n"
                f"    field_0x4=0x{self.field_0x4:x},\n"
                f"    field_0x8=0x{self.field_0x8:x},\n"
                f"    field_0xc=0x{self.field_0xc:x},\n"
                f"    field_0x10=0x{self.field_0x10:x},\n"
                f"    field_0x14=0x{self.field_0x14:x},\n"
                f"    field_0x18=0x{self.field_0x18:x},\n"
                f"    field_0x1c=0x{self.field_0x1c:x},\n"
                f"    field_0x20=0x{self.field_0x20:x},\n"
                f")"
            )

    class Hair(ctypes.Structure):
        _fields_ = [
            ("field_0x0", ctypes.c_uint32),
            ("intersection", HairIntersection),
            ("xs", ctypes.c_float * 3),
            ("ys", ctypes.c_float * 3),
            ("zs", ctypes.c_float * 3),
        ]

        def __repr__(self):
            return (
                f"{self.__class__.__name__}(\n"
                f"    field_0x0=0x{self.field_0x0:x},\n"
                f"    intersection={self.intersection},\n"
                f"    xs={[i for i in self.xs]},\n"
                f"    ys={[i for i in self.ys]},\n"
                f"    zs={[i for i in self.zs]},\n"
                f")"
            )

    hairs = (Hair * hair_group_header.hair_count)()
    bytes_read += file.readinto(hairs)
    print(f"hairs={pprint.pformat([i for i in hairs])}")

    return bytes_read


def parse_ctr_file(file_path: Path):
    with file_path.open("rb") as file:
        print(f"file_path=\"{file_path}\"")
        bytes_read = 0
        # As with all LHReleasedFiles, the contents are packed

        # In this case, only the magic string "LiOnHeAd" exists
        class FileHeader(ctypes.Structure):
            _fields_ = [
                ("magic", ctypes.c_char * 8),
            ]

            def __repr__(self):
                return f"{self.__class__.__name__}(magic={self.magic})"

        header = FileHeader()
        bytes_read += file.readinto(header)
        print(f"header={header}")

        # Right after is the section header with the name and size
        class SectionHeader(ctypes.Structure):
            _fields_ = [
                ("name", ctypes.c_char * 32),
                ("size", ctypes.c_uint32),
            ]

            def __repr__(self):
                return f"{self.__class__.__name__}(name={self.name}, size=0x{self.size:x})"

        section_header = SectionHeader()
        bytes_read += file.readinto(section_header)
        print(f"section_header={section_header}")

        # Save offset of section, after the header but before the contents
        section_offset = bytes_read

        # Then there is some metadata for morphable meshes
        class MorphableMeshSectionHeader(ctypes.Structure):
            _fields_ = [
                ("field_0x24", ctypes.c_uint32),  # TODO: unknown, is 0 for hand and 21 for creatures
                ("spec_version", ctypes.c_uint32),
                ("binary_version", ctypes.c_uint32),  # expected to be 6. lower than 4 is "old" code path
                ("base_mesh_name", ctypes.c_char * 32),
                ("morph_mesh_names", (ctypes.c_char * 32) * 6),  # 6 fixed length strings of 32 chars
            ]

            def __repr__(self):
                return (
                    f"{self.__class__.__name__}(\n"
                    f"    field_0x24=0x{self.field_0x24:x},\n"
                    f"    spec_version={self.spec_version},\n"
                    f"    binary_version={self.binary_version},\n"
                    f"    base_mesh_name={self.base_mesh_name},\n"
                    f"    morph_mesh_names={[i.value for i in self.morph_mesh_names]},\n"
                    f")"
                )

        morphable_header = MorphableMeshSectionHeader()
        bytes_read += file.readinto(morphable_header)
        print(f"morphable_header={morphable_header}")

        # Spec file contains animation categories and animation names
        # We can also use it to get the count of animations which we use to get the correct file offsets
        animation_specs = parse_spec_file(section_header.name.decode('ascii'), morphable_header.spec_version)
        print(f"animation_specs={pprint.pformat(animation_specs)}")
        animation_names = [i[0] for i in itertools.chain.from_iterable(animation_specs.values())]

        # After the header is the anim set, a variable length array of offsets relative to the section offset
        anim_offsets = (ctypes.c_uint32 * len(animation_names))()

        bytes_read += file.readinto(anim_offsets)
        print(f"anim_set=[{', '.join(['0x%08x' % i for i in anim_offsets])}]")

        # Following the animation offsets are chained offsets which can lead to extra data
        extra_offset = ctypes.c_uint32()
        bytes_read += file.readinto(extra_offset)
        print(f"extra_offset=0x{extra_offset.value:08x}")

        bytes_read += parse_animation(file, section_offset, animation_names, anim_offsets, "default")

        # save copy of base anim_offsets which is used to parse extra data
        base_anim_offsets = anim_offsets[:]

        # Creature files have different animations for the morph meshes (evil, good, thin, fat) weak, strong are skipped
        for name in morphable_header.morph_mesh_names[:4]:
            if not name.value:
                continue
            # Set file to next animation set
            file.seek(section_offset + extra_offset.value)
            bytes_read += file.readinto(anim_offsets)
            print(f"anim_set=[{', '.join(['0x%08x' % i for i in anim_offsets])}]")

            # Again, the get pointer to the next part
            extra_offset = ctypes.c_uint32()
            bytes_read += file.readinto(extra_offset)
            print(f"extra_offset=0x{extra_offset.value:08x}")

            bytes_read += parse_animation(file, section_offset, animation_names, anim_offsets, name.value)

        # Once all the animation sets are loaded, the extra offset points to hair groups data (even if there are none)
        if morphable_header.binary_version > 4:
            field_0x4830 = ctypes.c_uint32()
            bytes_read += file.readinto(field_0x4830)
            print(f"field_0x4830=0x{field_0x4830.value:08x}")
        hair_group_count = ctypes.c_int32()
        bytes_read += file.readinto(hair_group_count)
        print(f"hair_group_count={hair_group_count.value}")

        for _ in range(hair_group_count.value):
            bytes_read += parse_hair_group(file, section_offset)

        # The extra data segment is in relation to the number of animations in the base animation set
        class ExtraData(ctypes.Structure):
            _fields_ = [
                ("field_0x0", ctypes.c_uint32),
                ("field_0x4", ctypes.c_uint32),
                ("field_0x8", ctypes.c_uint32),
                ("field_0xc", ctypes.c_uint32),
            ]

            def __repr__(self):
                return (
                    f"{self.__class__.__name__}(\n"
                    f"    field_0x0=0x{self.field_0x0:08x},\n"
                    f"    field_0x4=0x{self.field_0x4:08x},\n"
                    f"    field_0x8=0x{self.field_0x8:08x},\n"
                    f"    field_0xc=0x{self.field_0xc:08x},\n"
                    f")"
                )

        for i, name in zip(base_anim_offsets, animation_names):
            if i == 0:
                continue
            assert(morphable_header.binary_version != 0)
            has_data = ctypes.c_uint32()
            bytes_read += file.readinto(has_data)
            extra_data = []
            while has_data:
                data = ExtraData()
                bytes_read += file.readinto(data)
                bytes_read += file.readinto(has_data)
                extra_data.append(data)
            print(f'{name}: extra_data={extra_data}')

        print("0x%X bytes read" % bytes_read)
        section_bytes_read = bytes_read - section_offset
        print("0x%X bytes read from section" % section_bytes_read)
        the_rest = file.read()
        print("0x%X bytes left in file" % len(the_rest))
        print("0x%X bytes left in section" % (section_header.size - section_bytes_read))
        print(the_rest)


# Make sure to run from black and white install dir (the dir which contains "Data/")

for path in Path("Data/CTR").iterdir():
    if path.is_file():
        try:
            parse_ctr_file(path)
        except Exception as ex:
            print("Failed to parse %s" % path)
            raise ex

# parse_ctr_file(Path("Data/CTR/bcow.cbn"))
# parse_ctr_file(Path("Data/CTR/bwolf.CBN"))
# parse_ctr_file(Path("Data/CTR/hh.HBN"))