CameraSprite.ts

import { Container, MIPMAP_MODES, SCALE_MODES, Sprite, Texture } from "pixi.js";
import SimpleSignal from "simplesignal";

import Config from "../../../config/Config";
import CameraUtils from "../../utils/CameraUtils";
import FaceTracker from "../../utils/FaceTracker";
import Box from "../shapes/Box";

export default class CameraSprite extends Container {

	public readonly onStart = new SimpleSignal<(cameraSprite: CameraSprite) => void>();
	public readonly onError = new SimpleSignal<(cameraSprite: CameraSprite) => void>();

	private _videoTexture?: Texture;
	private _videoSprite?: Sprite;

	private _video: HTMLVideoElement;
	private _faceTracker?: FaceTracker;

	private _isConnected: boolean = false;
	private _cover: Box;

	private __width: number = NaN;
	private __height: number = NaN;

	private _lastFacePosition?: { x: number, y: number, radius: number };


	// ================================================================================================================
	// CONSTRUCTOR ----------------------------------------------------------------------------------------------------

	constructor(width: number = 640, height: number = 480, trackFace: boolean) {
		super();

		// Binds
		this.handleCameraConnect = this.handleCameraConnect.bind(this);
		this.handleCameraConnectError = this.handleCameraConnectError.bind(this);
		this.handleVideoCanPlay = this.handleVideoCanPlay.bind(this);
		this.handleDetectedFace = this.handleDetectedFace.bind(this);

		// Internal settings
		this.__width = width;
		this.__height = height;

		// Create instances
		this._video = document.createElement("video");
		this._video.autoplay = true;
		this._video.muted = true;

		this._cover = new Box();
		this.addChild(this._cover);

		if (trackFace) {
			// The video element needs to be in the document for detection to work
			this._video.style.visibility = "hidden";
			document.documentElement.appendChild(this._video);

			this._faceTracker = new FaceTracker();
			this._faceTracker.onDetectedFace.add(this.handleDetectedFace);
			this._faceTracker.start(this._video, true, Config.Platform.IS_MOBILE ? 1 : 0);
		}

		// Events
		this.addEvents();

		// Finally, try connecting
		this.connect();
	}


	// ================================================================================================================
	// PUBLIC INTERFACE -----------------------------------------------------------------------------------------------

	public get width() {
		return this.__width;
	}

	public set width(value: number) {
		if (this.__width !== value) {
			this.__width = value;
			this.applySize();
		}
	}

	public get height() {
		return this.__height;
	}

	public set height(value: number) {
		if (this.__height !== value) {
			this.__height = value;
			this.applySize();
		}
	}

	public get isConnected() {
		return this._isConnected;
	}

	public destroy() {
		this.disconnect();
		this.removeEvents();
		this.onStart.removeAll();
		this.onError.removeAll();
		if (this._videoTexture) this._videoTexture.destroy();
		if (this._videoSprite) this._videoSprite.destroy();
		if (this._faceTracker) this._faceTracker.destroy();
		this._cover.destroy();
		super.destroy();
	}


	// ================================================================================================================
	// EVENT INTERFACE ------------------------------------------------------------------------------------------------

	private handleCameraConnect(stream: MediaStream) {
		this._video.srcObject = stream;
		this._video.play();
	}

	private handleCameraConnectError(error: any) {
		this.onError.dispatch(this);
	}

	private handleVideoCanPlay() {
		// TODO: this is overall a messy solution, but needed because Pixi can't handle Video textures
		// with MediaStreams yet. In the future we should probably create a low-level MediaStreamResource
		// and use it instead.
		// See discussion: http://www.html5gamedevs.com/topic/42086-using-webcam-with-pixijs-v4/
		this._videoTexture = Texture.from(this._video);
		this._videoTexture.baseTexture.scaleMode = SCALE_MODES.LINEAR;
		this._videoTexture.baseTexture.mipmap = MIPMAP_MODES.OFF;
		// The video is supposed to be able to auto-update constantly, but this isn't working,
		// so we "simulate" it by calling _onPlayStart() on the VideoResource
		(this._videoTexture.baseTexture.resource as any)._onPlayStart();
		this._videoSprite = new Sprite(this._videoTexture);
		this.addChild(this._videoSprite);
		this._videoSprite.mask = this._cover;
		this.applySize();

		this._isConnected = true;
		this.onStart.dispatch(this);
	}

	private handleDetectedFace(x: number, y: number, radius: number) {
		if (this._videoSprite) {
			this._lastFacePosition = { x, y, radius };
			this.applySize();
		}
	}


	// ================================================================================================================
	// PRIVATE INTERFACE ----------------------------------------------------------------------------------------------

	private addEvents() {
		CameraUtils.onConnectCamera.add(this.handleCameraConnect);
		CameraUtils.onConnectCameraError.add(this.handleCameraConnectError);
		this._video.addEventListener("canplay", this.handleVideoCanPlay, false);
	}

	private removeEvents() {
		CameraUtils.onConnectCamera.remove(this.handleCameraConnect);
		CameraUtils.onConnectCameraError.remove(this.handleCameraConnectError);
		this._video.removeEventListener("canplay", this.handleVideoCanPlay, false);
	}

	private connect() {
		CameraUtils.connectCamera(this.width, this.height);
	}

	private disconnect() {
		this._isConnected = false;
		CameraUtils.disconnectCamera();
	}

	private applySize() {
		const prevWidth = this._cover.width;
		const prevHeight = this._cover.height;

		// If the size has changed, just move it there immediately instead of attenuate
		const immediate = prevWidth !== this.width || prevHeight !== this.height;

		this._cover.width = this.width;
		this._cover.height = this.height;
		// TODO: allow changing camera stream size?
		if (this._videoSprite) {
			if (this._lastFacePosition) {
				const { x, y, radius } = this._lastFacePosition;
				const attenuation = immediate ? 1 : 8; // 1 = no attenuation, > 1 = some attenuation

				// Find best scale
				const desiredFaceWidth = 0.2 * this.width;
				const desiredFaceHeight = 0.3 * this.height;
				const desiredScaleX = desiredFaceWidth / radius;
				const desiredScaleY = desiredFaceHeight / radius;
				const desiredScale = (desiredScaleX + desiredScaleY) / 2;

				// Find best position
				const desiredFaceX = 0.5;
				const desiredFaceY = 0.4;

				const desiredX = this.width * desiredFaceX - x * desiredScale;
				const desiredY = this.height * desiredFaceY - y * desiredScale;

				this._videoSprite.scale.x = this._videoSprite.scale.y = this._videoSprite.scale.x + (desiredScale - this._videoSprite.scale.x) / attenuation;
				this._videoSprite.x += (desiredX - this._videoSprite.x) / attenuation;
				this._videoSprite.y += (desiredY - this._videoSprite.y) / attenuation;
			} else {
				this._videoSprite.x = 0;
				this._videoSprite.y = 0;
				this._videoSprite.width = this.width;
				this._videoSprite.height = this.height;
			}
		}
	}
}

CameraUtils.ts

import SimpleSignal from "simplesignal";

export default class CameraUtils {

	public static readonly onSourcesChanged = new SimpleSignal<(devices: MediaDeviceInfo[]) => void>();
	public static readonly onConnectCamera = new SimpleSignal<(stream: MediaStream) => void>();
	public static readonly onConnectCameraError = new SimpleSignal<(error: any) => void>();
	public static readonly onPermissionsChanged = new SimpleSignal<(denied: boolean, granted: boolean) => void>();

	private static _devices: MediaDeviceInfo[] = [];
	private static _stream: MediaStream | null = null;

	private static _permissionGranted?: boolean;
	private static _permissionDenied?: boolean;


	// ================================================================================================================
	// STATIC INITIALIZER ---------------------------------------------------------------------------------------------

	public static initialize() {
		if (this.supportsCamera) {
			navigator.mediaDevices.addEventListener("devicechange", this.handleDeviceChange.bind(this));
			this.updateDevices();
		}
		this.updatePermissions();
	}


	// ================================================================================================================
	// STATIC INTERFACE -----------------------------------------------------------------------------------------------

	public static connectCamera(width: number = 640, height: number = 480): boolean {
		if (this.supportsCamera) {
			const videoConstraints: MediaTrackConstraints = {
				width,
				height,
				facingMode: "user",
			};

			navigator.mediaDevices.getUserMedia({video: videoConstraints}).then((stream: MediaStream) => {
				this.updatePermissions();
				this._stream = stream;
				requestAnimationFrame(() => {
					this.onConnectCamera.dispatch(stream);
				});
			}).catch((error: any) => {
				this.updatePermissions();
				requestAnimationFrame(() => {
					this.onConnectCameraError.dispatch(error);
				});
			});
			return true;
		} else {
			return false;
		}
	}

	public static disconnectCamera() {
		if (this.isCameraConnected && this._stream) {
			const tracks = this._stream.getVideoTracks();
			tracks.forEach((t) => t.stop());
			this._stream = null;
		}
	}


	// ================================================================================================================
	// ACCESSOR INTERFACE ---------------------------------------------------------------------------------------------

	public static get supportsCamera() {
		return Boolean(navigator.mediaDevices && navigator.mediaDevices.enumerateDevices);
	}

	public static get hasVideoCamera() {
		return this.supportsCamera && this._devices.some((d) => d.kind === "videoinput");
	}

	public static get isCameraConnected() {
		return Boolean(this._stream);
	}

	public static get devices() {
		return this._devices.concat();
	}

	public static get streamSource() {
		return this._stream;
	}

	public static get permissionDenied() {
		return this._permissionDenied;
	}

	public static get permissionGranted() {
		return this._permissionGranted;
	}


	// ================================================================================================================
	// EVENT INTERFACE ------------------------------------------------------------------------------------------------

	private static handleDeviceChange() {
		this.updateDevices();
	}


	// ================================================================================================================
	// PRIVATE INTERFACE ----------------------------------------------------------------------------------------------

	private static updateDevices() {
		navigator.mediaDevices.enumerateDevices().then((devices) => {
			this._devices = devices.concat();
			this.onSourcesChanged.dispatch(this._devices);
		}).catch(() => {
			console.error("CameraUtils :: Cannot list cameras!");
		});
	}

	private static updatePermissions() {
		const permissions = (navigator as any).permissions;
		if (permissions && permissions.query) {
			permissions.query({name: "camera"}).then(({ state }: { state: string}) => {
				switch (state) {
					case "granted":
						this.setPermissions(false, true);
						break;
					case "denied":
						this.setPermissions(true, false);
						break;
					case "prompt":
						this.setPermissions(false, false);
						break;
					default:
						this.setPermissions(undefined, undefined);
						break;
				}
			});
		}
	}

	private static setPermissions(denied?: boolean, granted?: boolean) {
		if (denied !== this._permissionDenied || granted !== this._permissionGranted) {
			this._permissionDenied = denied;
			this._permissionGranted = granted;
			this.onPermissionsChanged.dispatch(this._permissionDenied, this._permissionGranted);
		}
	}
}

CameraUtils.initialize();