Prioritized raycaster for casting against 3D Physics components. https://twitter.com/shanecelis/status/1146822575978373120

PriorityPhysicsRaycaster.cs

/* Original code[1] Copyright (c) 2019 Shane Celis[2]
   Licensed under the MIT License[3]

   This comment generated by code-cite[4].

   [1]: https://gist.github.com/shanecelis/5379ed7c634222dc979e09e728ef0a6c
   [2]: https://github.com/shanecelis
   [3]: https://opensource.org/licenses/MIT
   [4]: https://github.com/shanecelis/code-cite
*/

using System.Collections.Generic;
using System.Linq;
using UnityEngine.UI;

namespace UnityEngine.EventSystems {
/// <summary>
/// Simple event system using prioritized physics raycasts.
/// </summary>
[AddComponentMenu("Event/Priority Physics Raycaster")]
[RequireComponent(typeof(Camera))]
/// <summary>
/// Prioritized raycaster for casting against 3D Physics components.
/// </summary>
public class PriorityPhysicsRaycaster : PhysicsRaycaster {

  [System.Serializable]
  protected class Entry {
    public LayerMask mask = kNoEventMaskSet;
    public int sortingOrder = 1;
  }

  [Header("Define masks that have higher priority.")]
  [SerializeField]
  protected Entry[] priorityMasks;

  protected PriorityPhysicsRaycaster() { }

  protected override void Awake() {
    // Check for issues.
    if (priorityMasks.Length == 0)
      Debug.LogWarning("No priority masks set. Consider setting or using regular PhysicsRaycaster.");

    for (int i = 0; i < priorityMasks.Length; i++) {
      // Log warning if it prioritizes layers that are not in the raycast eventMask.
      int mask = Normalize(priorityMasks[i].mask.value);
      int maskedValue = mask & Normalize(eventMask.value);
      if (maskedValue != mask) {
        var layers = MaskToBits(~maskedValue & mask)
          .Select(j => LayerToName(j))
          .ToArray();
        if (layers.Length == 0)
          throw new System.InvalidOperationException("Internal error. Layers should not be zero for priorityMask {mask} and eventMask {eventMask.value} but it is.");
        if (layers.Length == 1)
          Debug.LogWarning($"Priority mask {i} has a layer that is not in the eventMask: {layers[0]}.");
        else
          Debug.LogWarning($"Priority mask {i} has {layers.Length} layers that are not in the eventMask: {string.Join(", ", layers)}.");
      }

      // Log warning if it prioritizes everything.
      if (priorityMasks[i].mask.value == -1)
        Debug.LogWarning($"Priority mask {i} prioritizes everything.");

      // Log warning if it prioritizes nothing.
      if (priorityMasks[i].mask.value == 0)
        Debug.LogWarning($"Priority mask {i} prioritizes nothing.");

      if (priorityMasks[i].sortingOrder == 0)
        Debug.LogWarning($"Priority mask {i} sets the sorting order to 0: no priority, the default.");
    }
  }

  private static string LayerToName(int layer) {
    var str = LayerMask.LayerToName(layer);
    if (string.IsNullOrEmpty(str))
      return layer.ToString();
    else
      return layer + " " + str;
  }

  /** Account for the everything mask and turn it into a mask. */
  protected static int Normalize(int mask) {
    if (mask == -1) {
      // XXX: Why is the layerMask.value an int and not a uint? It doesn't seem
      // like it can represent layer 31, can it?

      //mask = ~0;
      mask = 0x7fffffff;
    }
    return mask;
  }

  protected static IEnumerable<int> MaskToBits(int mask) {
    mask = Normalize(mask);
    int i = 0;
    while (mask > 0 && i < 32) {
      if ((mask & 1) != 0)
        yield return i;
      i++;
      mask = mask >> 1;
    }
  }

  public override void Raycast(PointerEventData eventData, List<RaycastResult> resultAppendList) {
    base.Raycast(eventData, resultAppendList);
    if (priorityMasks.Length == 0)
      return;
    for (int i = 0; i < resultAppendList.Count; i++) {
      var r = resultAppendList[i];
      var layer = 1 << r.gameObject.layer;
      for (int j = 0; j < priorityMasks.Length; j++) {
        var mask = priorityMasks[j].mask.value;
        mask = mask == -1 ? 0x7fffffff : mask; // Handle everything layer.
        if ((layer & mask) != 0) {
          /*
            First thought was to reorder the resultAppendList,
            but EventSystem.RaycastComparer reorders all the RaycastResults.

            XXX: sortingOrder isn't used by the PhysicsRaycaster at all.
            sortingOrder is one criteria we're overloading to ensure our high
            priority layer comes out on top.
          */
          r.sortingOrder = priorityMasks[j].sortingOrder;
          resultAppendList[i] = r;
        }
      }
    }
  }
}
}