Unity Barracuda Depth sensing with fastdepth ONNX

DepthSensor.cs

using System.Linq;
using Unity.Barracuda;
using UnityEngine;
using UnityEngine.UI;

public class DepthSensor : MonoBehaviour
{
    [SerializeField] private NNModel _monoDepthONNX;
    [SerializeField] private RawImage _sourceImageView;
    [SerializeField] private RawImage _destinationImageView;
    private Model m_RuntimeModel;
    private IWorker worker;
    private WebCamTexture _webCamTexture;
    private RenderTexture outputRenderTexture;
    private int channelCount = 3;
    private TextureResizer.ResizeOptions _options;
    private TextureResizer _resizer;
    private RenderTexture frame;
    private Texture2D inputTexture;
    private Texture2D depthTexture;
    private Rect region;
    private int modelwidth = 224;
    private int modelheight = 224;
    

    private Vector3[] vertices;
    private int[] triangles;
    private Mesh mesh;
    private Color[] colors;

    private void Start()
    {
        InitWebCamFeed();
        InitBarracuda();
        InitResizerAndTextures();
        InitPointCloudMesh();
    }

    private void InitBarracuda()
    {
        m_RuntimeModel = ModelLoader.Load(_monoDepthONNX);
        worker = WorkerFactory.CreateComputeWorker(m_RuntimeModel);
    }

    private void InitWebCamFeed()
    {
        _webCamTexture = new WebCamTexture(620, 480, 30);
        _sourceImageView.texture = _webCamTexture;
        _webCamTexture.Play();
    }

    private void InitPointCloudMesh()
    {
        vertices = new Vector3[modelwidth * modelheight];
        triangles = MakeMeshTriangles();
        mesh = new Mesh();
        colors = new Color[modelwidth * modelheight];
    }

    private void InitResizerAndTextures()
    {
        _resizer = new TextureResizer();
        _options = new TextureResizer.ResizeOptions();
        _options.width = modelwidth;
        _options.height = modelheight;
        inputTexture = new Texture2D(modelwidth, modelheight, TextureFormat.RGB24, false);
        depthTexture = new Texture2D(modelwidth, modelheight, TextureFormat.RGB24, false);
        region = new Rect(0, 0, modelwidth, modelheight);
    }


    private void Update()
    {
        Color[] pixels =  _webCamTexture.GetPixels();

        if (pixels.Length >= (modelwidth * modelheight))
        {
            ResizeWebCamFeedToInputTexture();

            var tensor = new Tensor(inputTexture);
            // inference
            var output = worker.Execute(tensor).PeekOutput();
            float[] depth = output.AsFloats();
            PrepareDepthTextureFromFloats(depth);
            _destinationImageView.texture = depthTexture;
            
            UpdatePointCloudMeshFilter();
            tensor.Dispose();
        }
    }

    private void ResizeWebCamFeedToInputTexture()
    {
        //Resize the webcam texture into the input shape dimensions
        RenderTexture tex = _resizer.Resize(_webCamTexture, _options);
        RenderTexture.active = tex;
        inputTexture.ReadPixels(region, 0, 0);
        RenderTexture.active = null;
        inputTexture.Apply();
        _sourceImageView.texture = inputTexture;
    }

    private void PrepareDepthTextureFromFloats(float[] depth)
    {
        var min = depth.Min();
        var max = depth.Max();
        foreach (var pix in depth.Select((v, i) => new {v, i}))
        {
            var x = pix.i % modelwidth;
            var y = pix.i / modelwidth;
            var invY = modelheight - y - 1;

            // normalize depth value
            var val = (pix.v - min) / (max - min);
            depthTexture.SetPixel(x, y, new Color(val, 0.59f * val, 0.11f * val));
            var worldPos = new Vector3(x / (modelwidth / 0.9f), y / (modelheight / 0.9f), val);
            vertices[y * modelwidth + x] = worldPos;
            colors[y * modelwidth + x] = inputTexture.GetPixel(x, invY);
        }
        depthTexture.Apply();
    }

    private void UpdatePointCloudMeshFilter()
    {
        mesh.SetVertices(vertices);
        mesh.SetColors(colors);
        mesh.SetTriangles(triangles,0);
        mesh.SetIndices(mesh.GetIndices(0),MeshTopology.Points,0);
        GetComponent<MeshFilter>().sharedMesh = mesh;
    }

    public int[] MakeMeshTriangles()
    {
        var triangles = new int[(modelwidth - 1) * (modelheight - 1) * 6];
        for (int y = 0; y < modelheight - 1; ++y)
        {
            for (int x = 0; x < modelwidth - 1; ++x)
            {
                int ul = y * modelwidth + x;
                int ur = y * modelwidth + x + 1;
                int ll = (y + 1) * modelwidth + x;
                int lr = (y + 1) * modelwidth + x + 1;

                int offset = (y * (modelwidth - 1) + x) * 6;

                triangles[offset + 0] = ll;
                triangles[offset + 1] = ul;
                triangles[offset + 2] = ur;
                triangles[offset + 3] = ll;
                triangles[offset + 4] = ur;
                triangles[offset + 5] = lr;

            }
        }

        return triangles;
    }
}

Thanks

…

On Wed, Aug 4, 2021 at 11:19 AM zhouy546 ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ this is TextureResizer.cs using System.Collections; using System.Collections.Generic; using UnityEngine; public class TextureResizer { public RenderTexture Resize(WebCamTexture _webCamTexture, ResizeOptions _options) { RenderTexture dest = new RenderTexture(_options.width, _options.height,1,RenderTextureFormat.Default); Graphics.Blit(_webCamTexture, dest); return dest; } public RenderTexture TextureToRenderTexture(Texture texture,TextureFormat format) { Texture2D texture2D = new Texture2D(texture.width, texture.height, format, false); RenderTexture currentRT = RenderTexture.active; RenderTexture renderTexture = RenderTexture.GetTemporary(texture.width, texture.height, 32); Graphics.Blit(texture, renderTexture); RenderTexture.active = renderTexture; return renderTexture; } public Texture2D TextureToTexture2D(Texture texture) { Texture2D texture2D = new Texture2D(texture.width, texture.height, TextureFormat.RGBA32, false); RenderTexture currentRT = RenderTexture.active; RenderTexture renderTexture = RenderTexture.GetTemporary(texture.width, texture.height, 32); Graphics.Blit(texture, renderTexture); RenderTexture.active = renderTexture; texture2D.ReadPixels(new Rect(0, 0, renderTexture.width, renderTexture.height), 0, 0); texture2D.Apply(); RenderTexture.active = currentRT; RenderTexture.ReleaseTemporary(renderTexture); return texture2D; } public RenderTexture ConveretTexture2dIntoRenderTexture(in Texture2D input_texture2d) { // texRef is your Texture2D // You can also reduice your texture 2D that way RenderTexture rt = new RenderTexture(input_texture2d.width / 2, input_texture2d.height / 2, 0); RenderTexture.active = rt; // Copy your texture ref to the render texture Graphics.Blit(input_texture2d, rt); return rt; } public class ResizeOptions { public int width; public int height; } } — You are receiving this because you commented. Reply to this email directly, view it on GitHub <https://gist.github.com/05b5d2aebeb2d0db4907d18ab475ad50#gistcomment-3843875>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/ARAEAACNRJCR3JWQHFJQEZLT3DZXRANCNFSM4ZXNLO5Q> . Triage notifications on the go with GitHub Mobile for iOS <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android <https://play.google.com/store/apps/details?id=com.github.android&utm_campaign=notification-email> .