ReadyPlayerMe + FaceCap App

fun fun face funz.js

import { useFBX, useGLTF, OrbitControls as DOrbit } from "@react-three/drei";
import { useFrame, useLoader, useThree } from "@react-three/fiber";
import { Suspense, useEffect, useRef } from "react";
import {
  AnimationMixer,
  FileLoader,
  NumberKeyframeTrack,
  AnimationClip,
  NormalAnimationBlendMode,
  Audio,
  Color,
} from "three";
import { LightTemplate } from "../../canvas/LightTemplate/LightTemplate";
import { Starter } from "../../canvas/Starter/Starter";
import { Overlays } from "./Overlays";
import { Now } from "../../store/Now";
import { StarSkyGo } from "../../canvas/StarSkyGo/StarSkyGo";

let lok = {
  audioURL: "/facecap/bwahaha/FC_2021-11-22_15-54-36_bwa.mp3",
  facecapURL: `/facecap/bwahaha/FC_2021-11-22_15-54-36_bwa.txt`,
  avatarURL: `https://d1a370nemizbjq.cloudfront.net/08cf5815-ab1d-4b6f-ab5e-5ec1858ec885.glb`,
};

let mom = {
  audioURL: `/facecap/mom/FC_2021-11-22_16-2-56_mon.mp3`,
  facecapURL: `/facecap/mom/FC_2021-11-22_16-3-1_mom.txt`,
  avatarURL: `https://d1a370nemizbjq.cloudfront.net/9ed3ab34-25e0-4510-87a3-b820dffbf34d.glb`,
};

let asset = lok;

useGLTF.preload(asset.avatarURL);
useLoader.preload(FileLoader, asset.facecapURL);
useLoader.preload(FileLoader, asset.audioURL);

export default function FaceMo() {
  Now.makeKeyReactive("gameOverlay");
  Now.makeKeyReactive("faceCapAudio");
  return (
    <div className="h-full w-full">
      <Starter>
        <LightTemplate />
        <Background></Background>
        <StarSkyGo></StarSkyGo>

        {Now.gameOverlay === "play-game" && Now.faceCapAudio && (
          <Suspense fallback={null}>
            <FaceMomo></FaceMomo>
            <Cam></Cam>
          </Suspense>
        )}
      </Starter>
      <Overlays audioURL={asset.audioURL}></Overlays>
    </div>
  );
}

function Background() {
  let { scene } = useThree();
  useEffect(() => {
    let bg = new Color("#000000");
    scene.background = bg;
    return () => {
      scene.background = null;
    };
  }, [scene]);

  //
  return <group></group>;
}

function FaceMomo() {
  let glb = useGLTF(asset.avatarURL);
  let text = useLoader(FileLoader, asset.facecapURL);
  //
  /// FC_2021-11-22_15-39-47_lokfafa
  let render = useRef(() => {});

  //
  //
  useEffect(() => {
    let clean = () => {};
    let run = async () => {
      while (text.indexOf("_") !== -1) {
        text = text.replace(/_L/, "Left");
        text = text.replace(/_R/, "Right");
      }

      text = text.replace(
        `bs,browInnerUp,`,
        `bs,time,head.position.x,head.position.y,head.position.z,head.rotation.x,head.rotation.y,head.rotation.z,eyeLeft.rotation.x,eyeLeft.rotation.y,eyeRight.rotation.x,eyeRight.rotation.y,browInnerUp,`
      );

      let lines = text.split("\n");
      lines = lines.map((e) => {
        return e.split(",");
      });

      // let infoArr = lines.filter((a) => a[0] === "info");
      let keyframes = lines.filter((a) => a[0] === "k");
      let headers = lines.filter((a) => a[0] === "bs")[0];

      let mixer = new AnimationMixer(glb.scene);
      let headObj = glb.scene.getObjectByName("Wolf3D_Head");
      const expressions = Object.keys(headObj.morphTargetDictionary);
      const remain = Object.keys(headObj.morphTargetDictionary);
      // const others = [];
      // console.log(expressions);

      let tracks = [];
      let notsupport = [];
      let support = [];
      let lookups = [];
      headers.forEach((e) => {
        // if ()

        if (expressions.includes(e)) {
          support.push(e);
          tracks.push({
            morphName: e,
            type: "number",
            name: `Wolf3D_Head.morphTargetInfluences[${e}]`,
            times: [],
            values: [],
          });

          tracks.push({
            morphName: e,
            type: "number",
            name: `Wolf3D_Teeth.morphTargetInfluences[${e}]`,
            times: [],
            values: [],
          });

          remain.splice(
            remain.findIndex((ee) => e === ee),
            1
          );
        } else {
          if (false) {
          } else {
            notsupport.push(e);
          }
        }

        //
      });

      console.log("face not supported", remain);
      console.log("file not supported", notsupport);
      console.log("support", support);

      console.log("mouth", expressions);

      let keyFrameObjects = [];
      keyframes.forEach((kf) => {
        let obj = {};

        kf.forEach((v, i) => {
          let h = headers[i];
          if (h === "bs") {
            return;
          }

          // morpahName
          let trackForHeads = tracks.filter((e) => e.morphName === h);
          trackForHeads.forEach((trackForHead) => {
            if (trackForHead) {
              trackForHead.times.push(Number(kf[1]) * 0.001);
              trackForHead.values.push(Number(v));
            }
          });

          obj[h] = v;
        });

        keyFrameObjects.push(obj);
      });

      let trackLeftEyeX = {
        type: "euler",
        name: "EyeLeft.rotation[x]",
        times: [],
        values: [],
      };
      // let trackRightEye = {
      //   type: "euler",
      //   name: "EyeRight.rotation",
      //   times: [],
      //   values: [],
      // };

      // let mouthOpen = {
      //   type: "number",
      //   name: "Wolf3D_Teeth.morphTargetInfluences[jawOpen]",
      //   times: [],
      //   values: [],
      // };
      // let mouthOpen2 = {
      //   type: "number",
      //   name: "Wolf3D_Head.morphTargetInfluences[jawOpen]",
      //   times: [],
      //   values: [],
      // };

      // console.log(keyFrameObjects[0]);
      keyFrameObjects.forEach((kv) => {
        // mouthOpen.times.push(kv.time);
        // mouthOpen.values.push(kv["jawOpen"]);
        // mouthOpen2.times.push(kv.time);
        // mouthOpen2.values.push(kv["jawOpen"]);
        // trackRightEye;
      });

      let lastKeyframe = keyframes[keyframes.length - 1];
      let clipTracks = [
        //
        // new NumberKeyframeTrack(
        //   mouthOpen.name,
        //   new Float32Array(mouthOpen.times),
        //   new Float32Array(mouthOpen.values)
        // ),
      ];

      tracks.forEach((trackRaw) => {
        //
        if (trackRaw.times.length > 0) {
          clipTracks.push(
            new NumberKeyframeTrack(
              trackRaw.name,
              new Float32Array(trackRaw.times),
              new Float32Array(trackRaw.values)
            )
          );
        }
      });

      let faceClip = new AnimationClip(
        "facetrack",
        Number(lastKeyframe[1] * 0.001),
        clipTracks
      );

      // AnimationClip.CreateFromMorphTargetSequence("facetrack");

      let faceAction = mixer.clipAction(faceClip);
      faceAction.repetitions = 1;
      faceAction.clampWhenFinished = true;
      faceAction.play();
      Now.faceCapAudio.play();

      mixer.addEventListener("finished", () => {
        //
        //
      });

      /*
      mouthSmile -> "mouthSmileLeft", "mouthSmileRight"
      */

      // console.log(db, headers, keyframes, infoArr);

      // head position xyz, head eulerAngles xyz, left-eye eulerAngles xy, right-eye eulerAngles xy

      // let action = mixer.clipAction(fbx.animations[0]);
      // action.play();
      render.current = (dt) => {
        mixer.update(dt);
      };
      clean = () => {};
    };
    run();

    return () => {
      clean();
    };
  }, []);

  useFrame((st, dt) => {
    render.current(dt);
  });

  //

  return (
    <group>
      <directionalLight intensity={3} position={[0, 1, 0]} />
      <directionalLight intensity={2} position={[0, 1, 1]} />
      <directionalLight intensity={1} position={[0, 0, 1]} />
      <group position={[0, 0, 0]} rotation={[0, Math.PI * 0.05, 0]}>
        <primitive object={glb.scene}></primitive>
      </group>
      {/*  */}
      {/*  */}
      {/*  */}
      {/*  */}
    </group>
  );
}

function Cam() {
  let { get } = useThree();
  useEffect(() => {
    get().camera.position.y = 1.75;
    get().camera.position.z = 0.5;
    get().camera.lookAt(0, 1.75 * 0.94, 0);
  }, []);

  return (
    <group>
      <DOrbit target={[0, 1.75 * 0.94, 0]}></DOrbit>
    </group>
  );
}