Load gltf in pythreejs

load_model_from_gltf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pygltflib import GLTF2, Accessor, Skin, Node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download this file from https://paste.c-net.org/ScreechGrubby\n",
    "gltf = GLTF2.load(\"assets/X Bot.glb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from enum import IntEnum\n",
    "import struct\n",
    "\n",
    "class GLTFComponentType(IntEnum):\n",
    "    BYTE = 5120\n",
    "    UNSIGNED_BYTE = 5121\n",
    "    SHORT = 5122\n",
    "    UNSIGNED_SHORT = 5123\n",
    "    UNSIGNED_INT = 5125\n",
    "    FLOAT = 5126\n",
    "\n",
    "GLTF_COMPONENTTYPE_SIZES = {\n",
    "    GLTFComponentType.BYTE: 1,\n",
    "    GLTFComponentType.UNSIGNED_BYTE: 1,\n",
    "    GLTFComponentType.SHORT: 2,\n",
    "    GLTFComponentType.UNSIGNED_SHORT: 2,\n",
    "    GLTFComponentType.UNSIGNED_INT: 4,\n",
    "    GLTFComponentType.FLOAT: 4\n",
    "}\n",
    "\n",
    "GLTF_ACCESSORTYPE_COUNTS = {\n",
    "    \"SCALAR\": 1,\n",
    "    \"VEC2\": 2,\n",
    "    \"VEC3\": 3,\n",
    "    \"VEC4\": 4,\n",
    "    \"MAT2\": 4,\n",
    "    \"MAT3\": 9,\n",
    "    \"MAT4\": 16\n",
    "}\n",
    "\n",
    "GLTF_COMPONENT_UNPACK_FORMATS = {\n",
    "    GLTFComponentType.BYTE: \"b\",\n",
    "    GLTFComponentType.UNSIGNED_BYTE: \"B\",\n",
    "    GLTFComponentType.SHORT: \"h\",\n",
    "    GLTFComponentType.UNSIGNED_SHORT: \"H\",\n",
    "    GLTFComponentType.UNSIGNED_INT: \"I\",\n",
    "    GLTFComponentType.FLOAT: \"f\"\n",
    "}\n",
    "\n",
    "def get_dense_data(gltf: GLTF2, accessor: Accessor):\n",
    "    bufferView = gltf.bufferViews[accessor.bufferView]\n",
    "    buffer = gltf.buffers[bufferView.buffer]\n",
    "    buffer_data = gltf.get_data_from_buffer_uri(buffer.uri)\n",
    "    result = []\n",
    "    elem_stride = GLTF_ACCESSORTYPE_COUNTS[accessor.type] * GLTF_COMPONENTTYPE_SIZES[int(accessor.componentType)]\n",
    "    for i in range(accessor.count):\n",
    "        index = bufferView.byteOffset + accessor.byteOffset + i * elem_stride\n",
    "        base64_elem_data = buffer_data[index:index + elem_stride]\n",
    "        elem_data = struct.unpack(f\"<{GLTF_COMPONENT_UNPACK_FORMATS[accessor.componentType] * GLTF_ACCESSORTYPE_COUNTS[accessor.type]}\", base64_elem_data)\n",
    "        if len(elem_data) == 1:\n",
    "            result.append(elem_data[0])\n",
    "        else:\n",
    "            result.append(elem_data)\n",
    "            \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "primitive = gltf.meshes[gltf.scenes[gltf.scene].nodes[0]].primitives[0]\n",
    "\n",
    "indices = get_dense_data(gltf, gltf.accessors[primitive.indices])\n",
    "positions = get_dense_data(gltf, gltf.accessors[primitive.attributes.POSITION])\n",
    "normals = get_dense_data(gltf, gltf.accessors[primitive.attributes.NORMAL])\n",
    "texcoords = get_dense_data(gltf, gltf.accessors[primitive.attributes.TEXCOORD_0])\n",
    "joints = get_dense_data(gltf, gltf.accessors[primitive.attributes.JOINTS_0])\n",
    "joint_weights = get_dense_data(gltf, gltf.accessors[primitive.attributes.WEIGHTS_0])\n",
    "inverse_bind_matrices = get_dense_data(gltf, gltf.accessors[gltf.skins[0].inverseBindMatrices])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pythreejs import *\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rgba_to_html_color(rgba_tuple):\n",
    "    # Convert each RGBA value to its corresponding 8-bit integer representation\n",
    "    r = int(rgba_tuple[0] * 255)\n",
    "    g = int(rgba_tuple[1] * 255)\n",
    "    b = int(rgba_tuple[2] * 255)\n",
    "    a = int(rgba_tuple[3] * 255)\n",
    "\n",
    "    # Format the values as a hexadecimal color string\n",
    "    color_string = \"#{:02X}{:02X}{:02X}{:02X}\".format(r, g, b, a)\n",
    "\n",
    "    return color_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_material(material: Material):\n",
    "    result = MeshPhongMaterial(\n",
    "        color = rgba_to_html_color(material.pbrMetallicRoughness.baseColorFactor),\n",
    "        metallicFactor = material.pbrMetallicRoughness.metallicFactor,\n",
    "        roughnessFactor = material.pbrMetallicRoughness.roughnessFactor,\n",
    "        emissiveFactor = material.emissiveFactor,\n",
    "        skinning = True)\n",
    "\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "mesh_material = create_material(gltf.materials[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "mesh_geometry = BufferGeometry()\n",
    "mesh_geometry.attributes[\"position\"] = BufferAttribute(positions)\n",
    "mesh_geometry.attributes[\"index\"] = BufferAttribute(indices)\n",
    "mesh_geometry.attributes[\"uv\"] = BufferAttribute(texcoords)\n",
    "mesh_geometry.attributes[\"normal\"] = BufferAttribute(normals)\n",
    "mesh_geometry.attributes[\"skinIndex\"] = BufferAttribute(joints)\n",
    "mesh_geometry.attributes[\"skinWeight\"] = BufferAttribute(joint_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "\n",
    "def create_skeleton(gltf: GLTF2, skin: Skin, bones: List[Bone]):\n",
    "    def create_bone(gltf: GLTF2, node: Node):\n",
    "        bone = Bone(name = node.name, \n",
    "                    position = node.translation, \n",
    "                    scale = node.scale, \n",
    "                    quaternion = node.rotation)\n",
    "        if bones is not None:\n",
    "            bones.insert(0, bone)\n",
    "        for child_node in node.children:\n",
    "            child_bone = create_bone(gltf, gltf.nodes[child_node])\n",
    "            bone.add(child_bone)\n",
    "\n",
    "        return bone\n",
    "\n",
    "    return create_bone(gltf, gltf.nodes[skin.skeleton])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "bones = []\n",
    "root_bone = create_skeleton(gltf, gltf.skins[0], bones)\n",
    "skeleton = Skeleton(bones = bones, boneInverses = inverse_bind_matrices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "mesh = SkinnedMesh(mesh_geometry, mesh_material)\n",
    "mesh.add(root_bone)\n",
    "mesh.skeleton = skeleton\n",
    "\n",
    "helper = SkeletonHelper(mesh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "sphere = Mesh(\n",
    "    SphereBufferGeometry(1, 32, 16),\n",
    "    MeshStandardMaterial(color='red')\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c8293ea23cf64061be2e4ae2b5b95b9a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Renderer(camera=PerspectiveCamera(aspect=1.3333333333333333, position=(10.0, 6.0, 10.0), projectionMatrix=(1.0…"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "view_width = 800\n",
    "view_height = 600\n",
    "\n",
    "key_light = DirectionalLight(position=[0, 10, 10], intensity=0.6)\n",
    "ambient_light = AmbientLight()\n",
    "\n",
    "camera = PerspectiveCamera(position=[10, 6, 10], aspect=view_width/view_height)\n",
    "\n",
    "# This line doesn't work, draws a small white rectangle\n",
    "scene_elems = [camera, key_light, ambient_light, mesh, helper]\n",
    "\n",
    "# This line works, draws a red sphere\n",
    "# scene_elems = [camera, key_light, ambient_light, sphere]\n",
    "\n",
    "scene = Scene(children = scene_elems)\n",
    "renderer = Renderer(camera=camera, scene=scene,\n",
    "                     controls=[OrbitControls(controlling=camera)],\n",
    "                     width=view_width, height=view_height)\n",
    "renderer"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}