Crappy MS3D parser

matrix.py

import numpy
import math

def perspective(fovy, aspect, z_near, z_far):
    f = 1 / math.tan(math.radians(fovy) / 2)
    return numpy.array([[f / aspect,  0,                                   0,  0],
                        [          0, f,                                   0,  0],
                        [          0, 0, (z_far + z_near) / (z_near - z_far), -1],
                        [          0, 0, (2*z_far*z_near) / (z_near - z_far),  0]])

def look_at(eye, center, up):
    F = numpy.subtract(center, eye)
    f = F / numpy.linalg.norm(F)

    up_prime = up / numpy.linalg.norm(up)

    s = numpy.cross(f, up_prime)
    u = numpy.cross(s / numpy.linalg.norm(s), f)

    mat = numpy.identity(4)
    mat[:3, :3] = numpy.stack([s, u, -f], axis=1)

    return numpy.dot(mat, translate(-eye[0], -eye[1], -eye[2]))


def translate(x, y, z):
    mat = numpy.identity(4)
    mat[-1, :3] = x, y, z
    return mat

def scale(m, sx, sy, sz):
    return numpy.array([
        [sx,  0,  0, 0],
        [ 0, sy,  0, 0],
        [ 0,  0, sz, 0],
        [ 0,  0,  0, 1]
    ])

def rotate(angle, x, y, z):
    s = math.sin(math.radians(angle))
    c = math.cos(math.radians(angle))
    magnitude = math.sqrt(x*x + y*y + z*z)
    nc = 1 - c
      
    x /= magnitude
    y /= magnitude
    z /= magnitude

    return numpy.array([
        [     c + x**2 * nc, y * x * nc - z * s, z * x * nc + y * s, 0],
        [y * x * nc + z * s,      c + y**2 * nc, y * z * nc - x * s, 0],
        [z * x * nc - y * s, z * y * nc + x * s,      c + z**2 * nc, 0],
        [                 0,                  0,                  0, 1],
    ])

def rotate_x(angle):
    c = math.cos(angle)
    s = math.sin(angle)

    return numpy.array([
        [1, 0,  0, 0],
        [0, c, -s, 0],
        [0, s,  c, 0],
        [0, 0,  0, 1]
    ])

def rotate_y(angle):
    c = math.cos(angle)
    s = math.sin(angle)

    return numpy.array([
        [ c, 0, s, 0],
        [ 0, 1, 0, 0],
        [-s, 0, c, 0],
        [ 0, 0, 0, 1]
    ])

def rotate_z(angle):
    c = math.cos(angle)
    s = math.sin(angle)
    return numpy.array([
        [ c, -s, 0, 0],
        [ s,  c, 0, 0],
        [ 0,  0, 1, 0],
        [ 0,  0, 0, 1]
    ])

ms3d.py

#!/usr/bin/env python3
import struct
import matrix
import numpy
import math
import json

def unpack(fmt, fp_obj):
    return struct.unpack(fmt, fp_obj.read(struct.calcsize(fmt)))

def remove_null(s):
    if b'\x00' not in s:
        return None
    index = s.index(b'\x00')
    if index == 0: return None
    return bytes(s[:index]).decode('ascii')


def _build_joint_matrix(rotation, translation):
    rx, ry, rz = rotation
    x, y, z = translation

    translation_matrix = numpy.identity(4, dtype=numpy.float32)
    translation_matrix[:-1, -1] = translation


    rx_mat = matrix.rotate_x(rx)
    ry_mat = matrix.rotate_y(ry)
    rz_mat = matrix.rotate_z(rz)

    return translation_matrix.dot(rz_mat).dot(ry_mat).dot(rx_mat).T

def _build_inverted_joint_matrix(rotation, translation):
    rx, ry, rz = rotation
    x, y, z = translation

    rx_mat = matrix.rotate_x(-rx)
    ry_mat = matrix.rotate_y(-ry)
    rz_mat = matrix.rotate_z(-rz)

    translation_matrix = numpy.identity(4, dtype=numpy.float32)
    translation_matrix[:-1, -1] = (-x, -y, -z)

    return rx_mat.dot(ry_mat).dot(rz_mat).dot(translation_matrix).T


def lerp(s, e, t):
    return s + (e - s) * t

def rlerp(r, s, e):
    return (r - s) / (e - s)


header_struct = struct.Struct('<10sI')
vertex_struct = struct.Struct('<b3fBb')
triangle_struct = struct.Struct('<H3H9f6fBB')
material_struct = struct.Struct('<32s4f4f4f4fffb128s128s')
group_struct = struct.Struct('<B32sH')
joint_struct = struct.Struct('<b32s32s3f3f')
vertex_ex_v1_struct = struct.Struct('<3b3B')
vertex_ex_v2_struct = struct.Struct('<3b3BI')

class MS3DJoint(object):
    def __init__(self, name, parent, rotation, position, rotation_frames, translation_frames):
        self.name = name
        self.parent = parent
        self.rotation = rotation
        self.position = position
        self.children = []
        self.rotation_frames = rotation_frames
        self.translation_frames = translation_frames

        self.matrix = _build_joint_matrix(rotation, position)
        self.matrix_inverse = _build_inverted_joint_matrix(rotation, position)

    @staticmethod
    def from_file(fp):
        joint_data = joint_struct.unpack(fp.read(joint_struct.size))
        name, parent_name = map(remove_null, joint_data[1:1+2])

        rotation, position = (joint_data[3+n*3:3+n*3+3] for n in range(2))

        num_rotation_frames, num_translation_frames = unpack('<HH', fp)

        rotation_frames = []
        translation_frames = []

        for j in range(num_rotation_frames):
            time, = unpack('<f', fp)
            rot = unpack('<3f', fp)
            rotation_frames.append((time, rot))

        for j in range(num_translation_frames):
            time, = unpack('<f', fp)
            trans = unpack('<3f', fp)
            translation_frames.append((time, trans))

        return MS3DJoint(name, parent_name, rotation, position, rotation_frames, translation_frames)
        # self.joints.append(MS3DJoint(name, parent_name, rotation, position, rotation_frames, translation_frames))

    def matrix_for_frame(self, t):
        for (a_time, a_rot), (b_time, b_rot) in zip(self.rotation_frames, self.rotation_frames[1:]):
            if a_time < t < b_time: break

        T = rlerp(t, a_time, b_time)

        rotation = tuple(lerp(an, bn, T) for an, bn in zip(a_rot, b_rot))

        for (a_time, a_trans), (b_time, b_trans) in zip(self.translation_frames, self.translation_frames[1:]):
            if a_time < t < b_time: break

        translation = tuple(lerp(at, bt, T) for at, bt in zip(a_trans, b_trans))

        return _build_joint_matrix(rotation, translation)

class MS3DVertex(object):
    def __init__(self, vertex, bone_id):
        self.vertex = vertex
        self.bone_id = bone_id

    @staticmethod
    def from_file(fp):
        vertex_data = vertex_struct.unpack(fp.read(vertex_struct.size))
        vertex, bone_id = vertex_data[1:1+3], vertex_data[4]
        return MS3DVertex(vertex, bone_id)

class MS3DTriangle(object):
    def __init__(self, indices, normals, texcoords, smoothing_group=None, group_index=None):
        self.indices = indices
        self.normals = normals
        self.texcoords = texcoords

    @staticmethod
    def from_file(fp):
        triangle_data = triangle_struct.unpack(fp.read(triangle_struct.size))

        vertex_indices = triangle_data[1:1+3]
        normals = [triangle_data[4 + n*3:4 + n*3+3] for n in range(3)]
        texcoords = list(zip(*(triangle_data[13 + n*3:13 + n*3+3] for n in range(2))))

        return MS3DTriangle(vertex_indices, normals, texcoords)

class MS3DGroup(object):
    def __init__(self, name, triangles, material_index):
        self.name = name
        self.triangles = triangles
        self.material_index = material_index

    @staticmethod
    def from_file(fp):
        flags, name, n_triangles = group_struct.unpack(fp.read(group_struct.size))

        name = remove_null(name)

        triangle_indices = []
        for j in range(n_triangles):
            triangle_indices.append(unpack("<H", fp))

        mat_index = unpack("<b", fp)
        return MS3DGroup(name, triangle_indices, mat_index)


class MS3DComment(object):
    def __init__(self, comment):
        self.comment = comment

    @staticmethod
    def from_file(fp):
        index, comment_length = unpack('<ii', fp)
        comment = fp.read(comment_length)
        return MS3DComment(str(comment))

class MS3DMaterial(object):
    def __init__(self, name, ambient, diffuse, specular, emissive, shininess, transparency, mode, texture, alphamap):
        self.name = name
        self.ambient = ambient
        self.diffuse = diffuse
        self.specular = specular
        self.emissive = emissive
        self.shininess = shininess
        self.transparency = transparency
        self.mode = mode
        self.texture = texture
        self.alphamap = alphamap

    @staticmethod
    def from_file(fp):
        material_data = material_struct.unpack(fp.read(material_struct.size))
        name = remove_null(material_data[0])
        ambient, diffuse, specular, emissive = (
            material_data[1+n*4:1+n*4+4]
            for n in range(4)
        )

        shininess, transparency = material_data[17:17+2]
        mode = material_data[19]
        texture, alphamap = map(remove_null, material_data[20:20+2])
        return MS3DMaterial(
            name,
            ambient,
            diffuse,
            specular,
            emissive,
            shininess,
            transparency,
            mode,
            texture,
            alphamap
        )

class MS3DModel(object):
    def __init__(self, filename):
        self.joints = []
        self.group_comments = []
        self.material_comments = []
        self.joint_comments = []
        # There should only be 1 of these
        # but I'm storing them in an array because why not
        self.model_comments = []

        self.bone_ids = []
        self.vertex_weights = []

        with open(filename, 'rb') as fp:
            signature, version = header_struct.unpack(fp.read(header_struct.size))
            assert signature == b"MS3D000000", "Not a MS3D Model"

            num_vertices, = unpack("<H", fp)
            self.vertices = [MS3DVertex.from_file(fp) for _ in range(num_vertices)]

            num_triangles, = unpack("<H", fp)
            self.triangles = [MS3DTriangle.from_file(fp) for _ in range(num_triangles)]

            num_groups, = unpack('<H', fp)
            self.groups = [MS3DGroup.from_file(fp) for _ in range(num_groups)]

            num_materials, = unpack('<H', fp)
            self.materials = [MS3DMaterial.from_file(fp)]

            animation_fps, current_time, num_frames = unpack('<ffi', fp)

            num_joints, = unpack('<H', fp)
            self.joints = [MS3DJoint.from_file(fp) for _ in range(num_joints)]

            # I don't really care about this comment crap, But I Can't just skip over them
            # Because they are dynamically sized
            subversion, num_group_comments = unpack('<iI', fp)
            self.group_comments = [MS3DComment.from_file(fp)]

            num_material_comments, = unpack('<i', fp)
            self.material_comments = [MS3DComment.from_file(fp) for _ in range(num_material_comments)]

            num_joint_comments, = unpack('<i', fp)
            self.joint_comments = [MS3DComment.from_file(fp) for _ in range(num_joint_comments)]

            num_model_comments, = unpack('<i', fp)
            self.model_comments = [MS3DComment.from_file(fp) for _ in range(num_model_comments)]

            subversion, = unpack('<i', fp)

            for i in range(num_vertices):
                # For some reason v2 decided that weights should be in the range 0-100
                # instead of 0 - 255
                # IDK why you would discard that extra precision but WHATEVER
                if subversion == 1:
                    vertex_ex_data = vertex_ex_v1_struct.unpack(fp.read(vertex_ex_v1_struct.size))
                    bone_ids = vertex_ex_data[:3]
                    weights = tuple(weight / 255.0 for weight in vertex_ex_data[3:3+3])
                else:
                    vertex_ex_data = vertex_ex_v2_struct.unpack(fp.read(vertex_ex_v2_struct.size))
                    bone_ids = vertex_ex_data[:3]
                    weights = tuple(weight / 100.0 for weight in vertex_ex_data[3:3+3])

                w0, w1, w2 = weights
                w3 = 1.0 - w0 - w1 - w2

                b0 = self.vertices[i].bone_id
                b1, b2, b3 = bone_ids

                self.vertex_weights.append((w0, w1, w2, w3))
                self.bone_ids.append((b0, b1, b2, b3))

            # There's more crap I don't really care about after this point

        # Build up vertex buffers for use in OpenGL
        self.bone_id_buffer = numpy.array(self.bone_ids, dtype=numpy.int16)
        self.vertex_weight_buffer = numpy.array(self.vertex_weights, dtype=numpy.float32)

        self.vertex_buffer = numpy.array([
            self.vertices[n].vertex
            for triangle in self.triangles
            for n in triangle.indices
        ], dtype=numpy.float32)

        self.texcoord_buffer = numpy.array([
            texcoord
            for triangle in self.triangles
            for texcoord in triangle.texcoords
        ], dtype=numpy.float32)

        # TODO: Right now I am just generating a tree of joints
        # But I need to really have things like the inverse local
        # matrix and absolute matrix and a bunch of other crap
        # the tree is really only for debugging
        self.joint_dict = {joint.name: joint for joint in self.joints}

        self.joint_tree = []
        for joint in self.joints:
            if not joint.parent:
                self.joint_tree.append(joint)
            else:
                self.joint_dict[joint.parent].children.append(joint)



if __name__ == "__main__":
    from collections import defaultdict
    from argparse import ArgumentParser

    parser = ArgumentParser()
    parser.add_argument('fname', metavar='FILE')
    args = parser.parse_args()
    m = MS3DModel(args.fname)    
    result = defaultdict(list)
    for vert in m.vertices:
        result["vertices"].append(vert.vertex)

    for triangle in m.triangles:
        result["triangles"].append({
            "indices": triangle.indices,
            "normals": triangle.normals,
            "texcoords": triangle.texcoords
        })

    for joint in m.joints:
        result["joints"].append({
            "name": joint.name,
            "parent": joint.parent,
            "rotation": joint.rotation,
            "translation": joint.position,
            "rotation_frames": joint.rotation_frames,
            "translation_frames": joint.translation_frames,
        })

    for vertex_weights, bone_ids in zip(m.vertex_weights, m.bone_ids):
        result["bone_info"].append({
            "vertex_weights": vertex_weights,
            "bone_ids": bone_ids
        })
    
    print(json.dumps(result, sort_keys=True, indent=4, separators=(',', ': ')))