[Unity] PhysicsRaycaster for indirectly casting rays inside a quad that has a RenderTexture on it

.Raycast inside RenderTexture.gif

Intersection.cs

using UnityEngine;

public static class Intersection
{
    private const float EPSILON = 1e-8f;

    /// <summary>Returns the distance t to the triangle along the ray and the barycenter coordinates of the hit.
    /// Adapted from http://three-eyed-games.com/2019/03/18/gpu-path-tracing-in-unity-part-3/</summary>
    private static bool IntersectRayTriangle(Ray ray, Vector3 v0, Vector3 v1, Vector3 v2, bool ignoreBackfacing, out float t, out float u, out float v)
    {
        t = u = v = default;

        // Find vectors for two edges sharing v0
        Vector3 edge1 = v1 - v0;
        Vector3 edge2 = v2 - v0;

        // Begin calculating determinant - also used to calculate U parameter
        Vector3 pvec = Vector3.Cross(ray.direction, edge2);

        // If determinant is near zero, ray lies in plane of triangle
        float det = Vector3.Dot(edge1, pvec);

        // Use backface culling
        if (ignoreBackfacing && det < EPSILON)
            return false;
        float inv_det = 1.0f / det;

        // Calculate distance from v0 to ray origin
        Vector3 tvec = ray.origin - v0;

        // Calculate U parameter and test bounds
        u = Vector3.Dot(tvec, pvec) * inv_det;
        if (u < 0.0 || u > 1.0f)
            return false;

        // Prepare to test V parameter
        Vector3 qvec = Vector3.Cross(tvec, edge1);

        // Calculate V parameter and test bounds
        v = Vector3.Dot(ray.direction, qvec) * inv_det;
        if (v < 0.0 || u + v > 1.0f)
            return false;

        // calculate t, ray intersects triangle
        t = Vector3.Dot(edge2, qvec) * inv_det;

        return true;
    }

    /// <summary>Returns the distance t to the triangle along the ray and the barycenter coordinates of the hit.
    /// Adapted from http://three-eyed-games.com/2019/03/18/gpu-path-tracing-in-unity-part-3/</summary>
    private static bool IntersectRayTriangle(Ray ray, Vector3 v0, Vector3 v1, Vector3 v2, bool ignoreBackfacing, out float t, out float u, out float v, out float w)
    {
        var hit = IntersectRayTriangle(ray, v0, v1, v2, ignoreBackfacing, out t, out u, out v);
        w = 1 - u - v;
        return hit;
    }

    /// <summary>Returns the distance t to the quad along the ray and the barycenter coordinates of the hit.</summary>
    public static bool IntersectRayQuad(Ray ray, Vector3 v0, Vector3 v1, Vector3 v2, Vector3 v3, bool ignoreBackfacing, out float t, out Vector2 uv)
    {
        if (IntersectRayTriangle(ray, v0, v1, v2, ignoreBackfacing, out t, out var u, out var v, out var w))
        {
            //uv = u * Vector2.up + v * Vector2.one + w * Vector2.zero;
            // u(0,1) + v(1,1) + w(0,0)
            uv = new Vector2(v, u + v);
            return true;
        }

        if (IntersectRayTriangle(ray, v2, v3, v0, ignoreBackfacing, out t, out u, out v, out w))
        {
            //uv = u * Vector2.right + v * Vector2.zero + w * Vector2.one;
            // u(1,0) + v(0,0) + w(1,1)
            uv = new Vector2(u + w, w);
            return true;
        }

        t = default;
        uv = default;
        return false;
    }
}

RenderTextureImagePhysicsRaycaster.cs

using System.Collections.Generic;
using UnityEngine;
using UnityEngine.EventSystems;
using UnityEngine.UI;

/// <summary>PhysicsRaycaster that casts its rays inside a RenderTexture on a RawImage.
/// You can think of it as casting physics rays through a portal. Which sounds kinda cool.</summary>
public class RenderTextureImagePhysicsRaycaster : PhysicsRaycaster
{
    private class RaycastHitComparer : IComparer<RaycastHit>
    {
        public static RaycastHitComparer instance = new RaycastHitComparer();
        public int Compare(RaycastHit x, RaycastHit y)
        {
            return x.distance.CompareTo(y.distance);
        }
    }

    private static readonly Vector3[] _cornersCache = new Vector3[4];

    [Tooltip("The RawImage whose RectTransform will be used as the target quad for the raycast. It must have the same RenderTexture assigned as the renderTextureCamera below.")]
    public RawImage renderTextureRawImage;

    [Tooltip("The camera from which the physics raycasts will be made from. It must have the same RenderTexture assigned as the rawImage above below.")]
    public Camera renderTextureCamera;

    [Tooltip("If true, then the RawImage needs to be facing this camera in order to trigger the raycasts inside the RenderTexture.")]
    public bool ignoreBackfacing;

    private RaycastHit[] _hitsCache;

    public override void Raycast(PointerEventData eventData, List<RaycastResult> resultAppendList)
    {
        // This method is overridden for two reasons:
        // - We want to call our custom implementation of ComputeRayAndDistance, which is not virtual, thus the new keyword in its declaration.
        // - We want to use the LayerMask of the renderTextureCamera, not the eventCamera.

        Ray ray = new Ray();
        int displayIndex = 0;
        float distanceToClipPlane = 0;
        if (!ComputeRayAndDistance(eventData, ref ray, ref displayIndex, ref distanceToClipPlane))
            return;

        int hitCount = 0;

        var finalRTEventMask = (renderTextureCamera != null) ? renderTextureCamera.cullingMask & eventMask : -1;

        if (m_MaxRayIntersections == 0)
        {
            _hitsCache = Physics.RaycastAll(ray, distanceToClipPlane, finalRTEventMask);
            hitCount = _hitsCache.Length;
        }
        else
        {
            if (m_LastMaxRayIntersections != m_MaxRayIntersections)
            {
                _hitsCache = new RaycastHit[m_MaxRayIntersections];
                m_LastMaxRayIntersections = m_MaxRayIntersections;
            }

            hitCount = Physics.RaycastNonAlloc(ray, _hitsCache, distanceToClipPlane, finalRTEventMask);
        }

        if (hitCount != 0)
        {
            if (hitCount > 1)
                System.Array.Sort(_hitsCache, 0, hitCount, RaycastHitComparer.instance);

            for (int b = 0, bmax = hitCount; b < bmax; ++b)
            {
                resultAppendList.Add(
                    new RaycastResult
                    {
                        gameObject = _hitsCache[b].collider.gameObject,
                        module = this,
                        distance = _hitsCache[b].distance,
                        worldPosition = _hitsCache[b].point,
                        worldNormal = _hitsCache[b].normal,
                        screenPosition = eventData.position,
                        displayIndex = displayIndex,
                        index = resultAppendList.Count,
                        sortingLayer = 0,
                        sortingOrder = 0
                    }
                );
            }
        }
    }

    protected new bool ComputeRayAndDistance(PointerEventData eventData, ref Ray rayInRTSpace,
        ref int eventDisplayIndex, ref float distanceToClipPlane)
    {
        if (!ValidateReferences()) return false;

        var baseRay = new Ray();
        if (!base.ComputeRayAndDistance(eventData, ref baseRay, ref eventDisplayIndex, ref distanceToClipPlane))
        {
            return false;
        }

        // Use the ray returned by the base PhysicRaycaster to check collision against the RawImage's rect.
        var rectTransform = renderTextureRawImage.transform as RectTransform;
        rectTransform.GetWorldCorners(_cornersCache);
        if (!Intersection.IntersectRayQuad(baseRay, _cornersCache[0], _cornersCache[1], _cornersCache[2], _cornersCache[3], ignoreBackfacing, out var t, out var quadUV))
        {
            // The rect was not hit, so we discard this raycast by returning false.
            return false;
        }

        // Create the ray in the rtCamera's space that comes out of the point that was hit on the rect.
        var intersectionPoint = baseRay.origin + baseRay.direction * t;
        rayInRTSpace = renderTextureCamera.ViewportPointToRay(quadUV);

        eventDisplayIndex = renderTextureCamera.targetDisplay;

        // The base PhysicsRaycaster computes the distance to the eventCamera's clip planes, but we want to do the same
        // for our inner rtCamera instead.
        float projectionDirection = rayInRTSpace.direction.z;
        distanceToClipPlane = Mathf.Approximately(0.0f, projectionDirection)
            ? Mathf.Infinity
            : Mathf.Abs((renderTextureCamera.farClipPlane - renderTextureCamera.nearClipPlane) / projectionDirection);

        return true;
    }

    private bool ValidateReferences()
    {
        return renderTextureRawImage != null && renderTextureCamera != null &&
            renderTextureCamera != eventCamera &&
            renderTextureRawImage.mainTexture == renderTextureCamera.targetTexture;
    }
}

TestRenderTextureImagePhysicsRaycaster.unitypackage

I did this for a RawImage because it fits my needs. It should be easy to adapt this to work with a PlaneCollider or a quad in a MeshRenderer. It should also be easy to make this work with other types of raycasters, such as the GraphicRaycaster.

Check the included unitypackage for an example setup.