Unity Barracuda Depth sensing with fastdepth ONNX

DepthSensor.cs

using System.Linq;
using Unity.Barracuda;
using UnityEngine;
using UnityEngine.UI;

public class DepthSensor : MonoBehaviour
{
    [SerializeField] private NNModel _monoDepthONNX;
    [SerializeField] private RawImage _sourceImageView;
    [SerializeField] private RawImage _destinationImageView;
    private Model m_RuntimeModel;
    private IWorker worker;
    private WebCamTexture _webCamTexture;
    private RenderTexture outputRenderTexture;
    private int channelCount = 3;
    private TextureResizer.ResizeOptions _options;
    private TextureResizer _resizer;
    private RenderTexture frame;
    private Texture2D inputTexture;
    private Texture2D depthTexture;
    private Rect region;
    private int modelwidth = 224;
    private int modelheight = 224;
    

    private Vector3[] vertices;
    private int[] triangles;
    private Mesh mesh;
    private Color[] colors;

    private void Start()
    {
        InitWebCamFeed();
        InitBarracuda();
        InitResizerAndTextures();
        InitPointCloudMesh();
    }

    private void InitBarracuda()
    {
        m_RuntimeModel = ModelLoader.Load(_monoDepthONNX);
        worker = WorkerFactory.CreateComputeWorker(m_RuntimeModel);
    }

    private void InitWebCamFeed()
    {
        _webCamTexture = new WebCamTexture(620, 480, 30);
        _sourceImageView.texture = _webCamTexture;
        _webCamTexture.Play();
    }

    private void InitPointCloudMesh()
    {
        vertices = new Vector3[modelwidth * modelheight];
        triangles = MakeMeshTriangles();
        mesh = new Mesh();
        colors = new Color[modelwidth * modelheight];
    }

    private void InitResizerAndTextures()
    {
        _resizer = new TextureResizer();
        _options = new TextureResizer.ResizeOptions();
        _options.width = modelwidth;
        _options.height = modelheight;
        inputTexture = new Texture2D(modelwidth, modelheight, TextureFormat.RGB24, false);
        depthTexture = new Texture2D(modelwidth, modelheight, TextureFormat.RGB24, false);
        region = new Rect(0, 0, modelwidth, modelheight);
    }


    private void Update()
    {
        Color[] pixels =  _webCamTexture.GetPixels();

        if (pixels.Length >= (modelwidth * modelheight))
        {
            ResizeWebCamFeedToInputTexture();

            var tensor = new Tensor(inputTexture);
            // inference
            var output = worker.Execute(tensor).PeekOutput();
            float[] depth = output.AsFloats();
            PrepareDepthTextureFromFloats(depth);
            _destinationImageView.texture = depthTexture;
            
            UpdatePointCloudMeshFilter();
            tensor.Dispose();
        }
    }

    private void ResizeWebCamFeedToInputTexture()
    {
        //Resize the webcam texture into the input shape dimensions
        RenderTexture tex = _resizer.Resize(_webCamTexture, _options);
        RenderTexture.active = tex;
        inputTexture.ReadPixels(region, 0, 0);
        RenderTexture.active = null;
        inputTexture.Apply();
        _sourceImageView.texture = inputTexture;
    }

    private void PrepareDepthTextureFromFloats(float[] depth)
    {
        var min = depth.Min();
        var max = depth.Max();
        foreach (var pix in depth.Select((v, i) => new {v, i}))
        {
            var x = pix.i % modelwidth;
            var y = pix.i / modelwidth;
            var invY = modelheight - y - 1;

            // normalize depth value
            var val = (pix.v - min) / (max - min);
            depthTexture.SetPixel(x, y, new Color(val, 0.59f * val, 0.11f * val));
            var worldPos = new Vector3(x / (modelwidth / 0.9f), y / (modelheight / 0.9f), val);
            vertices[y * modelwidth + x] = worldPos;
            colors[y * modelwidth + x] = inputTexture.GetPixel(x, invY);
        }
        depthTexture.Apply();
    }

    private void UpdatePointCloudMeshFilter()
    {
        mesh.SetVertices(vertices);
        mesh.SetColors(colors);
        mesh.SetTriangles(triangles,0);
        mesh.SetIndices(mesh.GetIndices(0),MeshTopology.Points,0);
        GetComponent<MeshFilter>().sharedMesh = mesh;
    }

    public int[] MakeMeshTriangles()
    {
        var triangles = new int[(modelwidth - 1) * (modelheight - 1) * 6];
        for (int y = 0; y < modelheight - 1; ++y)
        {
            for (int x = 0; x < modelwidth - 1; ++x)
            {
                int ul = y * modelwidth + x;
                int ur = y * modelwidth + x + 1;
                int ll = (y + 1) * modelwidth + x;
                int lr = (y + 1) * modelwidth + x + 1;

                int offset = (y * (modelwidth - 1) + x) * 6;

                triangles[offset + 0] = ll;
                triangles[offset + 1] = ul;
                triangles[offset + 2] = ur;
                triangles[offset + 3] = ll;
                triangles[offset + 4] = ur;
                triangles[offset + 5] = lr;

            }
        }

        return triangles;
    }
}

Hi, TextureResizer class is missing

Can you please add it to the code.

Thanks in advance.

Hi, TextureResizer class is missing

Can you please add it to the code.

Thanks in advance.

I uploaded a unity project with the dependency sorted
https://github.com/paulorenanmelo/MonocularDepthSensing

this is TextureResizer.cs

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class TextureResizer
{

public RenderTexture Resize(WebCamTexture _webCamTexture, ResizeOptions _options)
{



    RenderTexture dest = new RenderTexture(_options.width, _options.height,1,RenderTextureFormat.Default);

    Graphics.Blit(_webCamTexture, dest);

    return dest;
}


public RenderTexture TextureToRenderTexture(Texture texture,TextureFormat format)
{
    Texture2D texture2D = new Texture2D(texture.width, texture.height, format, false);
    RenderTexture currentRT = RenderTexture.active;
    RenderTexture renderTexture = RenderTexture.GetTemporary(texture.width, texture.height, 32);
    Graphics.Blit(texture, renderTexture);

    RenderTexture.active = renderTexture;


    return renderTexture;
}

public Texture2D TextureToTexture2D(Texture texture)
{
    Texture2D texture2D = new Texture2D(texture.width, texture.height, TextureFormat.RGBA32, false);
    RenderTexture currentRT = RenderTexture.active;
    RenderTexture renderTexture = RenderTexture.GetTemporary(texture.width, texture.height, 32);
    Graphics.Blit(texture, renderTexture);

    RenderTexture.active = renderTexture;
    texture2D.ReadPixels(new Rect(0, 0, renderTexture.width, renderTexture.height), 0, 0);
    texture2D.Apply();

    RenderTexture.active = currentRT;
    RenderTexture.ReleaseTemporary(renderTexture);

    return texture2D;
}

public RenderTexture ConveretTexture2dIntoRenderTexture(in Texture2D input_texture2d)
{
    // texRef is your Texture2D
    // You can also reduice your texture 2D that way
    RenderTexture rt = new RenderTexture(input_texture2d.width / 2, input_texture2d.height / 2, 0);
    RenderTexture.active = rt;
    // Copy your texture ref to the render texture
    Graphics.Blit(input_texture2d, rt);

    return rt;
}


public class ResizeOptions {
    public int width;
    public int height;
}

}

Thanks

…

On Wed, Aug 4, 2021 at 11:19 AM zhouy546 ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ this is TextureResizer.cs using System.Collections; using System.Collections.Generic; using UnityEngine; public class TextureResizer { public RenderTexture Resize(WebCamTexture _webCamTexture, ResizeOptions _options) { RenderTexture dest = new RenderTexture(_options.width, _options.height,1,RenderTextureFormat.Default); Graphics.Blit(_webCamTexture, dest); return dest; } public RenderTexture TextureToRenderTexture(Texture texture,TextureFormat format) { Texture2D texture2D = new Texture2D(texture.width, texture.height, format, false); RenderTexture currentRT = RenderTexture.active; RenderTexture renderTexture = RenderTexture.GetTemporary(texture.width, texture.height, 32); Graphics.Blit(texture, renderTexture); RenderTexture.active = renderTexture; return renderTexture; } public Texture2D TextureToTexture2D(Texture texture) { Texture2D texture2D = new Texture2D(texture.width, texture.height, TextureFormat.RGBA32, false); RenderTexture currentRT = RenderTexture.active; RenderTexture renderTexture = RenderTexture.GetTemporary(texture.width, texture.height, 32); Graphics.Blit(texture, renderTexture); RenderTexture.active = renderTexture; texture2D.ReadPixels(new Rect(0, 0, renderTexture.width, renderTexture.height), 0, 0); texture2D.Apply(); RenderTexture.active = currentRT; RenderTexture.ReleaseTemporary(renderTexture); return texture2D; } public RenderTexture ConveretTexture2dIntoRenderTexture(in Texture2D input_texture2d) { // texRef is your Texture2D // You can also reduice your texture 2D that way RenderTexture rt = new RenderTexture(input_texture2d.width / 2, input_texture2d.height / 2, 0); RenderTexture.active = rt; // Copy your texture ref to the render texture Graphics.Blit(input_texture2d, rt); return rt; } public class ResizeOptions { public int width; public int height; } } — You are receiving this because you commented. Reply to this email directly, view it on GitHub <https://gist.github.com/05b5d2aebeb2d0db4907d18ab475ad50#gistcomment-3843875>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/ARAEAACNRJCR3JWQHFJQEZLT3DZXRANCNFSM4ZXNLO5Q> . Triage notifications on the go with GitHub Mobile for iOS <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android <https://play.google.com/store/apps/details?id=com.github.android&utm_campaign=notification-email> .