[Unity] Selection History Navigation

SelectionNavigation.cs

#if UNITY_EDITOR
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Runtime.InteropServices;
using System.Threading;
using UnityEditor;
using UnityEngine;

/// <summary>
/// Utility that listens to mouse forward/backward mous button input in the Unity editor to navigate
/// through a selection history.  This massively improves UX when jumping between different
/// GameObjects or assets, for example.  It can also be very useful when accidentally selecting
/// something else and quickly wanting to go back to what was selected before.
/// <para>
/// History is manually tracked using the selection changed event.  Since we want to grab the
/// input in regular edit mode, this is not using Unity's built-in input systems.
/// </para>
/// <para>
/// Currently only Windows is supported.
/// </para>
/// </summary>
internal sealed class SelectionNavigation : ScriptableObject
{
    [InitializeOnLoadMethod]
    private static void InitializeSelectionNavigation()
    {
        if (_instance != null)
        {
            return;
        }

        var instances = Resources.FindObjectsOfTypeAll<SelectionNavigation>();
        if (instances != null && instances.Length > 0)
        {
            return;
        }

        CreateInstance<SelectionNavigation>();
    }

    private enum Command
    {
        Forward,
        Backward
    }

    [Serializable]
    private struct SelectionHistoryEntry
    {
        public int[] value;
    }

    private const int MaxHistorySize = 128;

    private static SelectionNavigation _instance;
    private readonly ConcurrentQueue<Command> _queue = new ConcurrentQueue<Command>();
    [SerializeField] private List<SelectionHistoryEntry> selectionHistory;
    [SerializeField] private int selectionIndex;

    private void OnEnable()
    {
        if (_instance != null && _instance != this)
        {
            if (EditorApplication.isPlaying)
            {
                Destroy(this);
            }
            else
            {
                DestroyImmediate(this);
            }

            return;
        }

        _instance = this;
        hideFlags = HideFlags.DontUnloadUnusedAsset | HideFlags.DontSaveInEditor | HideFlags.DontSaveInBuild;
        DontDestroyOnLoad(this);

        if (!RegisterHook())
        {
            // No support for this platform, no need to track selection history.
            if (EditorApplication.isPlaying)
            {
                Destroy(this);
            }
            else
            {
                DestroyImmediate(this);
            }

            return;
        }

        AssemblyReloadEvents.beforeAssemblyReload += UnregisterHook;
        EditorApplication.update += ProcessQueue;

        Selection.selectionChanged += HandleSelectionChanged;
    }

    private void OnDisable()
    {
        UnregisterHook();

        while (_queue.TryDequeue(out _))
        {
            // Remove everything from queue. There's no clear on ConcurrentQueue due to implementation details.
        }

        // ReSharper disable DelegateSubtraction
        EditorApplication.update -= ProcessQueue;
        Selection.selectionChanged -= HandleSelectionChanged;
        // ReSharper restore DelegateSubtraction
    }

    private void HandleSelectionChanged()
    {
        ValidateState();

        var activeSelection = Selection.instanceIDs;
        var historySelection = selectionHistory[selectionIndex];
        if (activeSelection.Length == historySelection.value.Length)
        {
            var matches = true;
            for (var i = 0; i < historySelection.value.Length; i++)
            {
                if (activeSelection[i] != historySelection.value[i])
                {
                    matches = false;
                    break;
                }
            }

            // No actual change, ignore. This is at least the case when we changed the selection
            // to navigate the selection history.
            if (matches)
            {
                return;
            }
        }

        // Drop old future.
        if (selectionIndex < selectionHistory.Count - 1)
        {
            selectionHistory.RemoveRange(selectionIndex + 1, selectionHistory.Count - 1 - selectionIndex);
        }

        // Push state to history.
        selectionHistory.Add(new SelectionHistoryEntry {value = Selection.instanceIDs});
        selectionIndex = selectionHistory.Count - 1;

        // Trim history to avoid exceeding max length.
        if (selectionHistory.Count > MaxHistorySize)
        {
            selectionHistory.RemoveRange(0, selectionHistory.Count - MaxHistorySize);
        }
    }

    private void ProcessQueue()
    {
        ValidateState();

        while (_queue.TryDequeue(out var direction))
        {
            switch (direction)
            {
                case Command.Forward:
                    if (selectionIndex < selectionHistory.Count - 1)
                    {
                        selectionIndex++;
                        Selection.instanceIDs = selectionHistory[selectionIndex].value;
                    }

                    break;
                case Command.Backward:
                    if (selectionIndex > 0)
                    {
                        selectionIndex--;
                        Selection.instanceIDs = selectionHistory[selectionIndex].value;
                    }

                    break;
                default:
                    throw new ArgumentOutOfRangeException();
            }
        }
    }

    private void ValidateState()
    {
        if (selectionHistory == null)
        {
            selectionHistory = new List<SelectionHistoryEntry>();
        }

        if (selectionHistory.Count == 0)
        {
            selectionHistory.Add(new SelectionHistoryEntry {value = Selection.instanceIDs});
        }

        // Collapse empty entries -- new ones might be created when ids become invalid.
        var lastWasEmpty = false;
        for (var i = selectionHistory.Count - 1; i >= 0; i--)
        {
            var entry = selectionHistory[i];
            var isEmpty = true;
            for (var j = 0; j < entry.value.Length; j++)
            {
                if (EditorUtility.InstanceIDToObject(entry.value[j]) != null)
                {
                    isEmpty = false;
                    break;
                }
            }

            if (isEmpty)
            {
                if (lastWasEmpty)
                {
                    selectionHistory.RemoveAt(i);
                }
                else
                {
                    lastWasEmpty = true;
                }
            }
            else
            {
                lastWasEmpty = false;
            }
        }

        if (selectionIndex >= selectionHistory.Count)
        {
            selectionIndex = selectionHistory.Count - 1;
        }
    }

#if UNITY_EDITOR_WIN

    private static readonly Mutex HookMutex = new Mutex(false);
    private int _hookId;

    private bool RegisterHook()
    {
        if (_hookId == 0)
        {
#pragma warning disable 618
            _hookId = SetWindowsHookEx(WH_MOUSE, HandleMouseInput, IntPtr.Zero, AppDomain.GetCurrentThreadId());
#pragma warning restore 618
        }

        return true;
    }

    private void UnregisterHook()
    {
        if (_hookId != 0)
        {
            HookMutex.WaitOne();
            UnhookWindowsHookEx(_hookId);
            HookMutex.ReleaseMutex();
            _hookId = 0;
        }
    }

    #region Windows API Fun Land

    // ReSharper disable InconsistentNaming
    // ReSharper disable IdentifierTypo
    // ReSharper disable MemberCanBePrivate.Local
    // ReSharper disable FieldCanBeMadeReadOnly.Local

    private const int WH_MOUSE = 7;
    private const int WM_XBUTTONDOWN = 0x020B;
    private const int XBUTTON1 = 0x0001;
    private const int XBUTTON2 = 0x0002;
    private const int HC_ACTION = 0;

    [StructLayout(LayoutKind.Sequential)]
    private struct POINT
    {
        public int X;
        public int Y;
    }

    [StructLayout(LayoutKind.Sequential)]
    private struct MOUSEHOOKSTRUCT
    {
        public POINT pt;
        public IntPtr hwnd;
        public uint wHitTestCode;
        public IntPtr dwExtraInfo;
    }

    [StructLayout(LayoutKind.Sequential)]
    private struct MOUSEHOOKSTRUCTEX
    {
        public MOUSEHOOKSTRUCT mouseHookStruct;
        public int mouseData;
    }

    // ReSharper restore InconsistentNaming
    // ReSharper restore IdentifierTypo
    // ReSharper restore MemberCanBePrivate.Local
    // ReSharper restore FieldCanBeMadeReadOnly.Local

    private delegate int LowLevelMouseProc(int nCode, IntPtr wParam, IntPtr lParam);

    [DllImport("user32.dll", CharSet = CharSet.Auto, CallingConvention = CallingConvention.StdCall)]
    private static extern int SetWindowsHookEx(int idHook, LowLevelMouseProc lpFn, IntPtr hInstance, int threadId);

    [DllImport("user32.dll", CharSet = CharSet.Auto, CallingConvention = CallingConvention.StdCall)]
    private static extern bool UnhookWindowsHookEx(int idHook);

    [DllImport("user32.dll", CharSet = CharSet.Auto, CallingConvention = CallingConvention.StdCall)]
    private static extern int CallNextHookEx(int idHook, int nCode, IntPtr wParam, IntPtr lParam);

    private static int HandleMouseInput(int nCode, IntPtr wParam, IntPtr lParam)
    {
        HookMutex.WaitOne();
        try
        {
            try
            {
                if (nCode != HC_ACTION)
                {
                    return CallNextHookEx(_instance._hookId, nCode, wParam, lParam);
                }

                if (wParam.ToInt32() == WM_XBUTTONDOWN)
                {
                    var data = (MOUSEHOOKSTRUCTEX) Marshal.PtrToStructure(lParam, typeof(MOUSEHOOKSTRUCTEX));
                    switch (data.mouseData >> 16)
                    {
                        case XBUTTON1:
                            _instance._queue.Enqueue(Command.Backward);
                            return 1;
                        case XBUTTON2:
                            _instance._queue.Enqueue(Command.Forward);
                            return 1;
                    }
                }
            }
            catch
            {
                // Never throw exceptions out of hooks called from Windows.
            }

            return CallNextHookEx(_instance._hookId, nCode, wParam, lParam);
        }
        finally
        {
            HookMutex.ReleaseMutex();
        }
    }

    #endregion

#else
    private static bool RegisterHook()
    {
        // Unsupported platform.
        return false;
    }

    private static void UnregisterHook()
    {
        // Unsupported platform.
    }

#endif
}
#endif