VideoGUI starting point , edited from babylonjs ImageGui

VideoGui.ts

import { Nullable, Observable, serialize, _TypeStore } from "@babylonjs/core";
import { Control, Measure, Image } from "@babylonjs/gui";

/**
 * Class used to create 2D images
 */
export class VideoGui extends Control {
  private _workingCanvas: Nullable<HTMLCanvasElement> = null;

  private _domVideo!: HTMLVideoElement;
  private _imageWidth!: number;
  private _imageHeight!: number;
  private _loaded = false;
  private _stretch = Image.STRETCH_FILL;
  private _source!: Nullable<string>;
  private _autoScale = false;

  private _sourceLeft = 0;
  private _sourceTop = 0;
  private _sourceWidth = 0;
  private _sourceHeight = 0;
  private _svgAttributesComputationCompleted: boolean = false;
  private _isSVG: boolean = false;

  private _cellWidth: number = 0;
  private _cellHeight: number = 0;
  private _cellId: number = -1;

  private _detectPointerOnOpaqueOnly!: boolean;

  private _imageDataCache: {
    data: Uint8ClampedArray | null;
    key: string;
  } = { data: null, key: "" };

  /**
   * Observable notified when the content is loaded
   */
  public onImageLoadedObservable = new Observable<Image>();

  /**
   * Observable notified when _sourceLeft, _sourceTop, _sourceWidth and _sourceHeight are computed
   */
  public onSVGAttributesComputedObservable = new Observable<Image>();

  /**
   * Gets a boolean indicating that the content is loaded
   */
  public get isLoaded(): boolean {
    return this._loaded;
  }

  /**
   * Gets or sets a boolean indicating if pointers should only be validated on pixels with alpha > 0.
   * Beware using this as this will comsume more memory as the image has to be stored twice
   */
  @serialize()
  public get detectPointerOnOpaqueOnly(): boolean {
    return this._detectPointerOnOpaqueOnly;
  }

  public set detectPointerOnOpaqueOnly(value: boolean) {
    if (this._detectPointerOnOpaqueOnly === value) {
      return;
    }

    this._detectPointerOnOpaqueOnly = value;
  }

  /**
   * Gets or sets the left coordinate in the source image
   */
  @serialize()
  public get sourceLeft(): number {
    return this._sourceLeft;
  }

  public set sourceLeft(value: number) {
    if (this._sourceLeft === value) {
      return;
    }

    this._sourceLeft = value;

    this._markAsDirty();
  }

  /**
   * Gets or sets the top coordinate in the source image
   */
  @serialize()
  public get sourceTop(): number {
    return this._sourceTop;
  }

  public set sourceTop(value: number) {
    if (this._sourceTop === value) {
      return;
    }

    this._sourceTop = value;

    this._markAsDirty();
  }

  /**
   * Gets or sets the width to capture in the source image
   */
  @serialize()
  public get sourceWidth(): number {
    return this._sourceWidth;
  }

  public set sourceWidth(value: number) {
    if (this._sourceWidth === value) {
      return;
    }

    this._sourceWidth = value;

    this._markAsDirty();
  }

  /**
   * Gets or sets the height to capture in the source image
   */
  @serialize()
  public get sourceHeight(): number {
    return this._sourceHeight;
  }

  public set sourceHeight(value: number) {
    if (this._sourceHeight === value) {
      return;
    }

    this._sourceHeight = value;

    this._markAsDirty();
  }

  /**
   * Gets the image width
   */
  @serialize()
  public get imageWidth(): number {
    return this._imageWidth;
  }

  /**
   * Gets the image height
   */
  @serialize()
  public get imageHeight(): number {
    return this._imageHeight;
  }

  /** Indicates if the format of the image is SVG */
  public get isSVG(): boolean {
    return this._isSVG;
  }

  /** Gets the status of the SVG attributes computation (sourceLeft, sourceTop, sourceWidth, sourceHeight) */
  public get svgAttributesComputationCompleted(): boolean {
    return this._svgAttributesComputationCompleted;
  }

  /**
   * Gets or sets a boolean indicating if the image can force its container to adapt its size
   * @see https://doc.babylonjs.com/how_to/gui#image
   */
  @serialize()
  public get autoScale(): boolean {
    return this._autoScale;
  }

  public set autoScale(value: boolean) {
    if (this._autoScale === value) {
      return;
    }

    this._autoScale = value;

    if (value && this._loaded) {
      this.synchronizeSizeWithContent();
    }
  }

  /** Gets or sets the streching mode used by the image */
  @serialize()
  public get stretch(): number {
    return this._stretch;
  }

  public set stretch(value: number) {
    if (this._stretch === value) {
      return;
    }

    this._stretch = value;

    this._markAsDirty();
  }

  /**
   * Gets or sets the internal DOM image used to render the control
   */
  public set domVideo(value: HTMLVideoElement) {
    this._domVideo = value;
    this._loaded = false;
    this._imageDataCache.data = null;

    if (this._domVideo.width) {
      this._onVideoLoaded();
    } else {
      this._domVideo.onloadeddata = () => {
        this._onVideoLoaded();
      };
    }
  }

  public get domImage(): HTMLVideoElement {
    return this._domVideo;
  }

  private _onVideoLoaded(): void {
    this._imageDataCache.data = null;
    this._imageWidth = this._domVideo.width;

    this._imageHeight = this._domVideo.height;
    this._loaded = true;

    if (this._autoScale) {
      this.synchronizeSizeWithContent();
    }

    // this.onImageLoadedObservable.notifyObservers(this);

    this._markAsDirty();
  }

  /**
   * Gets the image source url
   */
  @serialize()
  public get source() {
    return this._source;
  }

  /**
   * Gets or sets image source url
   */
  public set source(value: Nullable<string>) {
    if (this._source === value) {
      return;
    }

    this._loaded = false;
    this._source = value;
    this._imageDataCache.data = null;

    if (value) {
      this._domVideo = document.createElement("video");
      this._domVideo.src = value;

      // testAppendVideo(this._domVideo, "speech bubble vid");
      this._domVideo.width = 540;
      this._domVideo.height = 404;
      this._domVideo.muted = true;
      this._domVideo.autoplay = true;
      this._domVideo.loop = true;
      this._domVideo.play();

      this._domVideo.onloadeddata = () => {
        this._onVideoLoaded();
      };
    }
    // if (value) {
    //   Tools.SetCorsBehavior(value, this._domImage);
    //   this._domImage.src = value;
    // }
  }

  /**
   * Gets or sets the cell width to use when animation sheet is enabled
   * @see https://doc.babylonjs.com/how_to/gui#image
   */
  get cellWidth(): number {
    return this._cellWidth;
  }
  set cellWidth(value: number) {
    if (this._cellWidth === value) {
      return;
    }

    this._cellWidth = value;
    this._markAsDirty();
  }

  /**
   * Gets or sets the cell height to use when animation sheet is enabled
   * @see https://doc.babylonjs.com/how_to/gui#image
   */
  get cellHeight(): number {
    return this._cellHeight;
  }
  set cellHeight(value: number) {
    if (this._cellHeight === value) {
      return;
    }

    this._cellHeight = value;
    this._markAsDirty();
  }

  /**
   * Gets or sets the cell id to use (this will turn on the animation sheet mode)
   * @see https://doc.babylonjs.com/how_to/gui#image
   */
  get cellId(): number {
    return this._cellId;
  }
  set cellId(value: number) {
    if (this._cellId === value) {
      return;
    }

    this._cellId = value;
    this._markAsDirty();
  }

  /**
   * Creates a new Image
   * @param name defines the control name
   * @param url defines the image url
   */
  constructor(public name?: string, url: Nullable<string> = null) {
    super(name);
    this.source = url;
  }

  /**
   * Tests if a given coordinates belong to the current control
   * @param x defines x coordinate to test
   * @param y defines y coordinate to test
   * @returns true if the coordinates are inside the control
   */
  public contains(x: number, y: number): boolean {
    if (!super.contains(x, y)) {
      return false;
    }

    if (!this._detectPointerOnOpaqueOnly || !this._workingCanvas) {
      return true;
    }
    return true;
  }

  protected _getTypeName(): string {
    return "Image";
  }

  /** Force the control to synchronize with its content */
  public synchronizeSizeWithContent() {
    if (!this._loaded) {
      return;
    }

    this.width = this._domVideo.width + "px";
    this.height = this._domVideo.height + "px";
  }

  protected _processMeasures(
    parentMeasure: Measure,
    context: CanvasRenderingContext2D
  ): void {
    if (this._loaded) {
      switch (this._stretch) {
        case Image.STRETCH_NONE:
          break;
        case Image.STRETCH_FILL:
          break;
        case Image.STRETCH_UNIFORM:
          break;
        case Image.STRETCH_NINE_PATCH:
          break;
        case Image.STRETCH_EXTEND:
          if (this._autoScale) {
            this.synchronizeSizeWithContent();
          }
          if (this.parent && this.parent.parent) {
            // Will update root size if root is not the top root
            this.parent.adaptWidthToChildren = true;
            this.parent.adaptHeightToChildren = true;
          }
          break;
      }
    }

    super._processMeasures(parentMeasure, context);
  }

  private _drawImage(
    context: CanvasRenderingContext2D,
    sx: number,
    sy: number,
    sw: number,
    sh: number,
    tx: number,
    ty: number,
    tw: number,
    th: number
  ) {
    context.drawImage(this._domVideo, sx, sy, sw, sh, tx, ty, tw, th);
  }

  public _draw(context: CanvasRenderingContext2D): void {
    context.save();

    if (this.shadowBlur || this.shadowOffsetX || this.shadowOffsetY) {
      context.shadowColor = this.shadowColor;
      context.shadowBlur = this.shadowBlur;
      context.shadowOffsetX = this.shadowOffsetX;
      context.shadowOffsetY = this.shadowOffsetY;
    }

    let x, y, width, height;
    if (this.cellId == -1) {
      x = this._sourceLeft;
      y = this._sourceTop;

      width = this._sourceWidth ? this._sourceWidth : this._imageWidth;
      height = this._sourceHeight ? this._sourceHeight : this._imageHeight;
    } else {
      let rowCount = this._domVideo.videoWidth / this.cellWidth;
      let column = (this.cellId / rowCount) >> 0;
      let row = this.cellId % rowCount;

      x = this.cellWidth * row;
      y = this.cellHeight * column;
      width = this.cellWidth;
      height = this.cellHeight;
    }

    this._applyStates(context);
    if (this._loaded) {
      switch (this._stretch) {
        case Image.STRETCH_NONE:
          this._drawImage(
            context,
            x,
            y,
            width,
            height,
            this._currentMeasure.left,
            this._currentMeasure.top,
            this._currentMeasure.width,
            this._currentMeasure.height
          );
          break;
        case Image.STRETCH_FILL:
          this._drawImage(
            context,
            x,
            y,
            width,
            height,
            this._currentMeasure.left,
            this._currentMeasure.top,
            this._currentMeasure.width,
            this._currentMeasure.height
          );
          break;
        case Image.STRETCH_UNIFORM:
          var hRatio = this._currentMeasure.width / width;
          var vRatio = this._currentMeasure.height / height;
          // var ratio = Math.min(hRatio, vRatio); // fit
          var ratio = Math.max(hRatio, vRatio); // cover
          var centerX = (this._currentMeasure.width - width * ratio) / 2;
          var centerY = (this._currentMeasure.height - height * ratio) / 2;

          this._drawImage(
            context,
            x,
            y,
            width,
            height,
            this._currentMeasure.left + centerX,
            this._currentMeasure.top + centerY,
            width * ratio,
            height * ratio
          );
          break;
        case Image.STRETCH_EXTEND:
          this._drawImage(
            context,
            x,
            y,
            width,
            height,
            this._currentMeasure.left,
            this._currentMeasure.top,
            this._currentMeasure.width,
            this._currentMeasure.height
          );
          break;
        // case Image.STRETCH_NINE_PATCH:
        //   this._renderNinePatch(context);
        //   break;
      }
    }

    context.restore();
  }

  public dispose() {
    // console.log("disposing video");

    super.dispose();
    // unloadVideo(this._domVideo);
    this.onImageLoadedObservable.clear();
    this.onSVGAttributesComputedObservable.clear();
  }

  // Static
  /** STRETCH_NONE */
  public static readonly STRETCH_NONE = 0;
  /** STRETCH_FILL */
  public static readonly STRETCH_FILL = 1;
  /** STRETCH_UNIFORM */
  public static readonly STRETCH_UNIFORM = 2;
  /** STRETCH_EXTEND */
  public static readonly STRETCH_EXTEND = 3;
  /** NINE_PATCH */
  // public static readonly STRETCH_NINE_PATCH = 4;
}
// _TypeStore.RegisteredTypes["BABYLON.GUI.Image"] = Image;