LipSync.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.Playables;
using UnityEngine.Timeline;
using System.Linq;
using VRM;

public class LipSync : MonoBehaviour
{
    private PlayableDirector m_director;
    private VRMBlendShapeProxy proxy;
    private TimelineClip currentClip;
    private double nextPlayTime;
    private int clipIndex;
    private List<TimelineClip> clips;
    private bool waitFlag = false;

    public GameObject avatar;

    const float THRESHOLD = 0.03f;
    const float INTERVAL = 0.05f;
    

    // Start is called before the first frame update
    void Start()
    {
        m_director = GetComponent<PlayableDirector>();
        proxy = avatar.GetComponent<VRMBlendShapeProxy>();
        currentClip = null;
        nextPlayTime = -1.0f;
        clipIndex = -1;

		IEnumerable<TrackAsset> tracks = (m_director.playableAsset as TimelineAsset).GetOutputTracks();
		TrackAsset track = tracks.FirstOrDefault(x => x.name == "Audio Track");
        if (track != null)
        {
            clips = track.GetClips().ToList();
        }
    }

    // Update is called once per frame
    void Update()
    {
        if(waitFlag) return;

        Debug.Log(nextPlayTime);

        double currentTime = m_director.time;

        if(currentClip == null && nextPlayTime < 0) {
            if(clipIndex + 1 >= clips.Count) return;
            nextPlayTime = clips[clipIndex + 1].start;
            return;
        } else if(currentClip == null && currentTime >= nextPlayTime) {
            if(clipIndex + 1 >= clips.Count) return;
            currentClip = clips[++clipIndex];
        } else if(currentClip == null) {
            return;
        }

        var audioClip = ((AudioPlayableAsset) currentClip.asset).clip;
        float[] allSamples = new float[audioClip.samples * audioClip.channels];
        double elapsedTime = currentTime - nextPlayTime;
        int freq = audioClip.frequency; // [Hz]
        if(elapsedTime * freq > allSamples.Length) {
            currentClip = null;
            if(clipIndex + 1 >= clips.Count) return;
            nextPlayTime = clips[clipIndex + 1].clipIn;
        }
        audioClip.GetData(allSamples, 0);
        
        StartCoroutine(changeMTH(allSamples[(int)(elapsedTime*freq)] > THRESHOLD ? 1 : 0));
        // Debug.Log($"currentVol:{allSamples[(int)(elapsedTime*freq)]}");
    }

    IEnumerator changeMTH(int open) {
        waitFlag = true;
        proxy.SetValue(BlendShapePreset.A, open);
        if(open > 0) {
            yield return new WaitForSeconds(INTERVAL);
            waitFlag = false;
        }
        waitFlag = false;
        yield return null;
    }   
}