Integrating NatDevice and OpenCV for Unity with a camera device output that streams camera images into an OpenCV matrix.

MatOutput.cs

/* 
*   NatDevice
*   Copyright (c) 2022 NatML Inc. All Rights Reserved.
*/

namespace NatSuite.Devices.Outputs {

    using System;
    using OpenCVForUnity.CoreModule;
    using OpenCVForUnity.UtilsModule;
    using Unity.Collections.LowLevel.Unsafe;

    /// <summary>
    /// Camera device output that converts camera images into OpenCV matrices.
    /// </summary>
    public sealed class MatOutput : IDisposable {

        #region --Client API--
        /// <summary>
        /// Pixel buffer output used to convert images to pixel buffers before uploading to Mat.
        /// </summary>
        public readonly PixelBufferOutput pixelBufferOutput;

        /// <summary>
        /// OpenCV matrix containing the latest camera image.
        /// </summary>
        public Mat matrix { get; private set; }

        /// <summary>
        /// Create a Mat output.
        /// </summary>
        /// <param name="pixelBufferOutput">Pixel buffer output used to convert images to pixel buffers.</param>
        public MatOutput (PixelBufferOutput pixelBufferOutput = null) {
            this.pixelBufferOutput = pixelBufferOutput ?? new PixelBufferOutput();
        }

        /// <summary>
        /// Update the output with a new camera image.
        /// </summary>
        public unsafe void Update (CameraImage image) {
            pixelBufferOutput.Update(image);
            var (width, height) = (pixelBufferOutput.width, pixelBufferOutput.height);
            matrix ??= new Mat(height, width, CvType.CV_8UC4);
            if (matrix.width() != width || matrix.height() != height)
                matrix.create(height, width, CvType.CV_8UC4);
            MatUtils.copyToMat((IntPtr)pixelBufferOutput.pixelBuffer.GetUnsafeReadOnlyPtr(), matrix);
        }

        /// <summary>
        /// Dispose the Mat output and release resources.
        /// </summary>
        public void Dispose () {
            matrix?.Dispose();
            pixelBufferOutput.Dispose();
        }
        #endregion


        #region --Operations--

        public static implicit operator Action<CameraImage> (MatOutput output) => output.Update;
        #endregion
    }
}

MatOutputTest.cs

/* 
*   NatDevice
*   Copyright (c) 2022 NatML Inc. All Rights Reserved.
*/

namespace NatSuite.Devices.Tests {

    using UnityEngine;
    using UnityEngine.UI;
    using NatSuite.Devices;
    using NatSuite.Devices.Outputs;
    using OpenCVForUnity.UnityUtils;

    sealed class MatOutputTest : MonoBehaviour {

        [Header(@"OpenCV")]
        [SerializeField] bool flip;

        [Header(@"UI")]
        [SerializeField] RawImage rawImage;
        [SerializeField] AspectRatioFitter aspectFitter;

        MediaDeviceQuery query;
        CameraDevice cameraDevice;
        MatOutput matOutput;
        Texture2D texture;

        async void Start () {
            // Request permissions
            var permissionStatus = await MediaDeviceQuery.RequestPermissions<CameraDevice>();
            if (permissionStatus != PermissionStatus.Authorized) {
                Debug.LogError(@"User did not grant camera permissions");
                return;
            }
            // Get camera device
            query = new MediaDeviceQuery(MediaDeviceCriteria.CameraDevice);
            cameraDevice = query.current as CameraDevice;
            // Start streaming
            matOutput = new MatOutput();
            cameraDevice.StartRunning(OnCameraImage);
        }

        void OnCameraImage (CameraImage image) {
            // Update the mat output
            matOutput.Update(image);
            // Convert to texture for display
            texture ??= new Texture2D(matOutput.matrix.width(), matOutput.matrix.height(), TextureFormat.RGBA32, false);
            Utils.fastMatToTexture2D(matOutput.matrix, texture, flip);
            // Display
            rawImage.texture = texture;
            aspectFitter.aspectRatio = (float)texture.width / texture.height;
        }

        void OnDisable () {
            if (cameraDevice?.running ?? false)
                cameraDevice.StopRunning();
            matOutput?.Dispose();
            Texture2D.Destroy(texture);
        }
    }
}