frame transformer with wasm

frame-transformer-js-wasm.js

const W = 640;   // video frame width
const H = 360;   // video frame height
const pixelData = await imageLoader("./seaside.png");  // get the pixel data of the backgroung image in RGBA format 

const buffer = new Uint8Array(W * H * 1.5); // byte buffer for the incoming frame in YUV 422 format
const bufferRGB = new Uint8Array(W * H * 4); // byte buffer for the result frame in RGBA format

const transformer = new TransformStream({
    async transform(videoFrame, controller) {
      const copyResult = await videoFrame.copyTo(buffer);

      const { stride, offset: Voffset } = copyResult[1];
      const { offset: Uoffset } = copyResult[2];

      // the WebAssembly function is called here, it uses the previously allocated buffers
      // the "instance" object keeps the reference for the compiled WebAssembly module. 
      // We will discuss the compiling and JS binding later
      instance.exports.process(
        stride,
        Voffset,
        Uoffset
      );
      
      const init = {
        timestamp: videoFrame.timestamp,
        codedWidth: W,
        codedHeight: H,
        format: "RGBA",
      };
      const newFrame = new VideoFrame(bufferRGB, init);  //  construct an RGBA frame

      videoFrame.close();
      controller.enqueue(newFrame);
    },
  });