FaceCap App and ReadyPlayerMe

facecap-adatpter.js

function FaceMomo() {
  // ready-player-me
  let glb = useGLTF(`/facecap/4cbc677b-3c53-4787-b62e-288d84f379a0.glb`);
  let text = useLoader(
    FileLoader,
    // from facecap
    `/facecap/FC_2021-11-18_11-4-48_testing.txt`
  );
  let render = useRef(() => {});
  //
  useEffect(() => {
    let clean = () => {};
    let run = async () => {
      while (text.indexOf("_") !== -1) {
        text = text.replace(/_L/, "Left");
        text = text.replace(/_R/, "Right");
      }

      text = text.replace(
        `bs,browInnerUp,`,
        `bs,time,head.position.x,head.position.y,head.position.z,head.rotation.x,head.rotation.y,head.rotation.z,eyeLeft.rotation.x,eyeLeft.rotation.y,eyeRight.rotation.x,eyeRight.rotation.y,browInnerUp,`
      );

      let lines = text.split("\n");
      lines = lines.map((e) => {
        return e.split(",");
      });

      // let infoArr = lines.filter((a) => a[0] === "info");
      let keyframes = lines.filter((a) => a[0] === "k");
      let headers = lines.filter((a) => a[0] === "bs")[0];

      let mixer = new AnimationMixer(glb.scene);
      let headObj = glb.scene.getObjectByName("Wolf3D_Head");
      const expressions = Object.keys(headObj.morphTargetDictionary);
      const remain = Object.keys(headObj.morphTargetDictionary);
      // const others = [];
      // console.log(expressions);

      let tracks = [];
      let notsupport = [];
      let support = [];
      let lookups = [];
      headers.forEach((e) => {
        // if ()

        if (expressions.includes(e)) {
          support.push(e);
          tracks.push({
            morphName: e,
            type: "number",
            name: `Wolf3D_Head.morphTargetInfluences[${e}]`,
            times: [],
            values: [],
          });

          tracks.push({
            morphName: e,
            type: "number",
            name: `Wolf3D_Teeth.morphTargetInfluences[${e}]`,
            times: [],
            values: [],
          });

          remain.splice(
            remain.findIndex((ee) => e === ee),
            1
          );
        } else {
          if (false) {
          } else {
            notsupport.push(e);
          }
        }

        //
      });

      console.log("face not supported", remain);
      console.log("file not supported", notsupport);
      console.log("support", support);

      console.log("mouth", expressions);

      let keyFrameObjects = [];
      keyframes.forEach((kf) => {
        let obj = {};

        kf.forEach((v, i) => {
          let h = headers[i];
          if (h === "bs") {
            return;
          }

          // morpahName
          let trackForHeads = tracks.filter((e) => e.morphName === h);
          trackForHeads.forEach((trackForHead) => {
            if (trackForHead) {
              trackForHead.times.push(Number(kf[1]) * 0.001);
              trackForHead.values.push(Number(v));
            }
          });

          obj[h] = v;
        });

        keyFrameObjects.push(obj);
      });

      let trackLeftEyeX = {
        type: "euler",
        name: "EyeLeft.rotation[x]",
        times: [],
        values: [],
      };
      // let trackRightEye = {
      //   type: "euler",
      //   name: "EyeRight.rotation",
      //   times: [],
      //   values: [],
      // };

      // let mouthOpen = {
      //   type: "number",
      //   name: "Wolf3D_Teeth.morphTargetInfluences[jawOpen]",
      //   times: [],
      //   values: [],
      // };
      // let mouthOpen2 = {
      //   type: "number",
      //   name: "Wolf3D_Head.morphTargetInfluences[jawOpen]",
      //   times: [],
      //   values: [],
      // };

      // console.log(keyFrameObjects[0]);
      keyFrameObjects.forEach((kv) => {
        // mouthOpen.times.push(kv.time);
        // mouthOpen.values.push(kv["jawOpen"]);
        // mouthOpen2.times.push(kv.time);
        // mouthOpen2.values.push(kv["jawOpen"]);
        // trackRightEye;
      });

      let lastKeyframe = keyframes[keyframes.length - 1];
      let clipTracks = [
        // new NumberKeyframeTrack(
        //   mouthOpen.name,
        //   new Float32Array(mouthOpen.times),
        //   new Float32Array(mouthOpen.values)
        // ),
      ];

      tracks.forEach((trackRaw) => {
        //
        if (trackRaw.times.length > 0) {
          clipTracks.push(
            new NumberKeyframeTrack(
              trackRaw.name,
              new Float32Array(trackRaw.times),
              new Float32Array(trackRaw.values)
            )
          );
        }
      });

      let faceClip = new AnimationClip(
        "facetrack",
        Number(lastKeyframe[1] * 0.001),
        clipTracks
      );

      // AnimationClip.CreateFromMorphTargetSequence("facetrack");

      let faceAction = mixer.clipAction(faceClip);
      faceAction.repetitions = 1;
      faceAction.clampWhenFinished = true;
      faceAction.play();
      Now.faceCapAudio.play();

      mixer.addEventListener("finished", () => {
        //
        //
      });

      /*
      mouthSmile -> "mouthSmileLeft", "mouthSmileRight"
      */

      // console.log(db, headers, keyframes, infoArr);

      // head position xyz, head eulerAngles xyz, left-eye eulerAngles xy, right-eye eulerAngles xy

      // let action = mixer.clipAction(fbx.animations[0]);
      // action.play();
      render.current = (dt) => {
        mixer.update(dt);
      };
      clean = () => {};
    };
    run();

    return () => {
      clean();
    };
  }, []);

  useFrame((st, dt) => {
    render.current(dt);
  });
  return (
    <group>
      <directionalLight intensity={3} position={[0, 1, 0]} />
      <directionalLight intensity={2} position={[0, 1, 1]} />
      <directionalLight intensity={1} position={[0, 0, 1]} />
      <primitive object={glb.scene}></primitive>
      {/*  */}
      {/*  */}
      {/*  */}
      {/*  */}
    </group>
  );
}