TheatreJS keyframes and WebMMuxer video encoding

renderer.ts

import { createRafDriver, ISheet, val } from '@theatre/core'
import WebMMuxer from 'webm-muxer'

export const rafDriver = createRafDriver({ name: 'Hubble rAF driver' })

export const useRenderer = ({
  sheet,
  fps = 30,
  bitrate = 1e6,
}: {
  sheet: ISheet
  fps?: number
  bitrate?: number
}) => {
  const { sequence } = sheet
  const duration = val(sequence.pointer.length)
  const totalFrames = duration * fps

  const startCapture = async ({ canvas }: { canvas: HTMLCanvasElement }) => {
    let i = 0
    let videoEncoder = new VideoEncoder({
      output: (chunk, meta) => muxer.addVideoChunk(chunk, meta, i * fps * 1000),
      error: (e) => console.error(e),
    })

    videoEncoder.configure({
      codec: 'vp09.00.10.08',
      width: 1280,
      height: 720,
      bitrate,
    })

    async function encodeFrame(data: VideoFrame) {
      const keyFrame = i % 60 === 0
      videoEncoder.encode(data, { keyFrame })
    }

    async function finishEncoding() {
      await videoEncoder.flush()
      muxer.finalize()
      reader.releaseLock()
      await fileWritableStream.close()
    }

    const fileHandle = await window.showSaveFilePicker({
      suggestedName: `video.webm`,
      types: [
        {
          description: 'Video File',
          accept: { 'video/webm': ['.webm'] },
        },
      ],
    })

    const fileWritableStream = await fileHandle.createWritable()

    const muxer = new WebMMuxer({
      target: fileWritableStream,
      video: {
        codec: 'V_VP9',
        width: 1280,
        height: 720,
        frameRate: fps,
      },
    })

    await sheet.project.ready
    const track = canvas.captureStream(0).getVideoTracks()[0]
    // @ts-expect-error
    const mediaProcessor = new MediaStreamTrackProcessor(track)
    const reader = mediaProcessor.readable.getReader()

    for (i = 0; i < totalFrames; i++) {
      const simTime = i / fps
      sequence.position = simTime
      rafDriver.tick(performance.now())

      console.log(`capturing frame ${i}/${totalFrames} at simTime ${simTime}`)

      // Simulate slow render
      await new Promise((resolve) => setTimeout(resolve, 100))

      // @ts-expect-error
      track.requestFrame()
      const result = await reader.read()
      const frame = result.value

      await encodeFrame(frame)

      frame.close()
    }
    finishEncoding()
  }

  return { startCapture }
}