ImageAccessVideoEffect.cs

using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using Windows.Foundation.Collections;
using Windows.Graphics.DirectX.Direct3D11;
using Windows.Media.Effects;
using Windows.Media.MediaProperties;


//---------------------------------------------------------
// How to use it in your application:
//---------------------------------------------------------
/*
    // start
    await m_MediaCaptureMgr.StartPreviewAsync().AsTask();

    // add our custom effect
    // prepare interface
    if (m_ImageAccessInterface != null)
    {
        m_ImageAccessInterface.NewTexture -= OnNewTexture;
        m_ImageAccessInterface.ProcessFrame -= OnProcessFrame;
    }
    m_ImageAccessInterface = new ImageAccessVideoEffectInterface();
    {
        m_ImageAccessInterface.NewTexture += OnNewTexture;
        m_ImageAccessInterface.ProcessFrame += OnProcessFrame;
    }
    PropertySet configuration = new PropertySet();
    configuration.Add("Interface", m_ImageAccessInterface);

    // attach the effect
    IMediaExtension videoEffect = await m_MediaCaptureMgr.AddVideoEffectAsync(
        new VideoEffectDefinition(typeof(ImageAccessVideoEffect).FullName, configuration),
        Windows.Media.Capture.MediaStreamType.VideoPreview);
*/


namespace OSLayerWindowsRTC
{
    public sealed class ImageAccessVideoEffectInterface
    {
        public Int64 NativeInputTexture { get; set; }

        /// <summary>
        /// Is invoked for incoming frames with a not yet seen input texture.
        /// The argument passed is a shared handle to be used with d3device.OpenSharedResource<Texture2D>(new IntPtr(sharedHandle));
        /// It is possible that the event is invoked every frame with alternating handles (double buffering or maybe more).
        /// Make sure you deal with this, e.g., by creating a Dictionary<Int64, SharpDX.Direct3D11.Texture2D>.
        /// </summary>
        public event EventHandler<Int64> NewTexture;

        /// <summary>
        /// Is invoked for incoming frames.
        /// The argument passed is a shared handle that was announced by the NewTexture event.
        /// Use the passed handle to access your dictionary and process image.
        /// </summary>
        public event EventHandler<Int64> ProcessFrame; 

        internal void InvokeNewTexture(Int64 sharedHandle) => NewTexture?.Invoke(this, sharedHandle);
        internal void InvokeProcessFrame(Int64 sharedHandle) => ProcessFrame?.Invoke(this, sharedHandle);
    }

    /// <summary>
    /// Custom Video Effect that allows to access the camera image from shaders without the of coyping using the CPU.
    /// A similar effect can be created to implement an GPU-based video effect that alters the images instead of just reading it.
    /// In this case, you don't need to pass the texture handles to the application but insead implement everything within the effect class.
    /// This class has be in a RTC library!
    /// </summary>
    public sealed class ImageAccessVideoEffect : IBasicVideoEffect
    {
        private class TextureInterfaceData
        {
            public SharpDX.Direct3D11.Texture2D LocalTexture;
            public SharpDX.Direct3D11.Texture2D SharedTexture;
            public Int64 SharedHandle;
        }

        /// <summary>
        /// Interface to the application.
        /// </summary>
        public ImageAccessVideoEffectInterface Interface
        {
            get
            {
                // fetch object from property set
                if (m_Interface == null)
                {
                    object val;
                    if (m_PropertySet != null && m_PropertySet.TryGetValue("Interface", out val))
                        m_Interface = (ImageAccessVideoEffectInterface) val;
                }

                // return cached object
                return m_Interface;
            }
        }
        private ImageAccessVideoEffectInterface m_Interface = null;

        /// <summary>
        /// Set from outside when creating the effect.
        /// </summary>
        /// <param name="configuration"></param>
        public void SetProperties(IPropertySet configuration) { m_PropertySet = configuration; }
        private IPropertySet m_PropertySet;


        // config of the effect
        public bool IsReadOnly => true;
        public MediaMemoryTypes SupportedMemoryTypes => MediaMemoryTypes.Gpu;
        public bool TimeIndependent => true;
        public IReadOnlyList<VideoEncodingProperties> SupportedEncodingProperties
        {
            get
            {
                var encodingProperties = new VideoEncodingProperties();
                encodingProperties.Subtype = "ARGB32";
                return new List<VideoEncodingProperties>() { encodingProperties };
            }
        }

        // d3d resources resouces 
        private SharpDX.Direct3D11.Device m_Device;
        private SharpDX.Direct3D11.DeviceContext m_Context;
        private Dictionary<IntPtr, TextureInterfaceData> m_InputTextures = new Dictionary<IntPtr, TextureInterfaceData>(16);

        public void SetEncodingProperties(VideoEncodingProperties encodingProperties, IDirect3DDevice device)
        {
            // based on code from the microsoft hololens sdk
            InteropStatics.IDirect3DDxgiInterfaceAccess interfaceAccess = device as InteropStatics.IDirect3DDxgiInterfaceAccess;
            IntPtr pResource = interfaceAccess.GetInterface(InteropStatics.ID3D11Device);

            if ((null == m_Device) || (m_Device.NativePointer != pResource))
            {
                m_Device = new SharpDX.Direct3D11.Device(pResource);
                m_Context = m_Device.ImmediateContext;
            }

            Marshal.Release(pResource);
        }

        public void ProcessFrame(ProcessVideoFrameContext context)
        {
            // based on code from the microsoft hololens sdk
            InteropStatics.IDirect3DDxgiInterfaceAccess interfaceAccess = context.InputFrame.Direct3DSurface as InteropStatics.IDirect3DDxgiInterfaceAccess;

            // looks like the resource is not shared..
            // if it would be, it would be basically this:
            /*
            IntPtr pResource = interfaceAccess.GetInterface(InteropStatics.IDXGIResource);
            var dxgiResouce = new SharpDX.DXGI.Resource(pResource);
            var sharedHandle = dxgiResouce.SharedHandle;    // can be used with d3device.OpenSharedResource<Texture2D>(sharedHandle);
            Marshal.Release(pResource);
            */

            // so we need to create a shared texture for this device and copy the image into it
            IntPtr pResource = interfaceAccess.GetInterface(InteropStatics.ID3D11Resource);
            if (!m_InputTextures.ContainsKey(pResource))
            {
                // keep track of all resources belonging to this texture
                TextureInterfaceData data = new TextureInterfaceData();
                data.LocalTexture = new SharpDX.Direct3D11.Texture2D(pResource);

                // create shared texture
                var sharedTextureDesc = data.LocalTexture.Description;
                sharedTextureDesc.OptionFlags = SharpDX.Direct3D11.ResourceOptionFlags.Shared;
                data.SharedTexture = new SharpDX.Direct3D11.Texture2D(m_Device, sharedTextureDesc);

                // get a handle to the shared texture
                using (var dxgiSurface = data.SharedTexture.QueryInterface<SharpDX.DXGI.Resource>())
                {
                    data.SharedHandle = dxgiSurface.SharedHandle.ToInt64();
                }

                // store info
                m_InputTextures.Add(pResource, data);
                Interface?.InvokeNewTexture(data.SharedHandle);
            }
            var key = pResource; // in case Marshel.Release will set the pointer to zero
            Marshal.Release(pResource);

            // copy content
            TextureInterfaceData current = m_InputTextures[key];
            m_Context.CopyResource(current.LocalTexture, current.SharedTexture);
            m_Context.Flush();

            // notify the application about the new frame
            Interface?.InvokeProcessFrame(current.SharedHandle);
        }

        public void Close(MediaEffectClosedReason reason)
        {
            // dispose all created resource the decrement ref counter
            foreach (var data in m_InputTextures.Values)
            {
                data.SharedTexture.Dispose();
                data.LocalTexture.Dispose();
            }
            m_InputTextures.Clear();

            m_Device?.Dispose();
            m_Context?.Dispose();
        }

        public void DiscardQueuedFrames()
        {
            // reset history of the effect
        }
    }

    internal static class InteropStatics
    {
        // can be found here:
        // https://github.com/lifthrasiir/w32api-directx-standalone/blob/master/include/d3d11.idl
        // https://github.com/lifthrasiir/w32api-directx-standalone/blob/master/include/dxgi.idl

        public static Guid ID3D11Resource = new Guid("dc8e63f3-d12b-4952-b47b-5e45026a862d");
        public static Guid ID3D11Device = new Guid("db6f6ddb-ac77-4e88-8253-819df9bbf140");
        public static Guid IDXGISurface = new Guid("cafcb56c-6ac3-4889-bf47-9e23bbd260ec");
        public static Guid IDXGIResource = new Guid("035f3ab4-482e-4e50-b41f-8a7f8bd8960b");

        [ComImport]
        [Guid("A9B3D012-3DF2-4EE3-B8D1-8695F457D3C1")]
        [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
        [ComVisible(true)]
        public interface IDirect3DDxgiInterfaceAccess : IDisposable
        {
            IntPtr GetInterface([In] ref Guid iid);
        };
    }
}