A selector palette implemented using Unity's PreviewRenderUtility

_doc.md

Part of a level-design tool made for Stereo Boy. This code has been tested with Unity version 2021.3.6f1.

You should be able to plop an instance of ObjectPalette into an EditorWindow. Here's a usage example:

public class SelectorWindow : EditorWindow {
  ObjectPalette _palette;
  
  private void OnDisable() {
    _palette.Cleanup();
    _palette = null;
  }

  private void OnGUI() {
    if (_palette = null) {
      _palette = new ObjectPalette(
        10, // number of columns
        selectorWidgetPrefab // prefab with BlockEditorPaletteSelector, containing grid + hover/selection indicators
      );
     
      // Populate the palette with objects
      _palette.AddPrefab(
        selectableObject,
        Vector3.zero, // display offset, if any
        Quaternion.identity // rotation, if any
      );
      
      // Add more selectable objects
      ...
    }
    
    _palette.Update(
      rect, // rect within the current window to draw into
      selectedIndex 
    )
  }
}

BlocEditorPaletteSelector.cs

using UnityEngine;

// This needs to live in your game code, not your editor code.
//
// Attach this to a prefab that contains:
// - a grid object whose upper left corner is at the origin
// - a child object (`hoverDisplay`) containing a hover indicator
// - a child object (`selectedDisplay`) containing a selection indicator
public class BlockEditorPaletteSelector : MonoBehaviour {
    public GameObject hoverDisplay;
    public GameObject selectedDisplay;
}

ObjectPalette.cs

using UnityEditor;
using UnityEngine;

public class ObjectPalette {
    private readonly int _objectsPerRow;

    // see: https://github.com/CyberFoxHax/Unity3D_PreviewRenderUtility_Documentation/wiki/PreviewRenderUtility
    private readonly PreviewRenderUtility _previewScene;
    private readonly BlockEditorPaletteSelector _blockEditorPaletteSelector;

    private int _numObjects;
    private bool _firstDraw = true;

    private const float CellWorldSize = 1;
    private const float ObjectScale = 0.6f * CellWorldSize;
    private const float ObjectYPos = 0.5f * ObjectScale;
    private const float CameraXRot = 45;

    public ObjectPalette(
        int objectsPerRow,
        GameObject blockPaletteSelectorPrefab
    ) {
        _objectsPerRow = objectsPerRow;

        _previewScene = new PreviewRenderUtility(true, true) {
            camera = {
                orthographic = true,
                orthographicSize = 1f,
                nearClipPlane = 0,
                farClipPlane = 50f,
                transform = {
                    rotation = Quaternion.Euler(CameraXRot, 0, 0),
                },
            },
            ambientColor = new Color(1f, 1f, 1f, 0),
        };

        _numObjects = 0;

        var selectorObj =
            _previewScene.InstantiatePrefabInScene(blockPaletteSelectorPrefab);
        _blockEditorPaletteSelector = selectorObj.GetComponent<BlockEditorPaletteSelector>();
    }

    public GameObject AddPrefab(GameObject prefab, Vector3 offset, Quaternion rot) {
        var obj = _previewScene.InstantiatePrefabInScene(prefab);
        obj.transform.position = GetItemPos(_numObjects) + offset;
        obj.transform.rotation = rot;
        obj.transform.localScale *= ObjectScale;

        _numObjects += 1;

        return obj;
    }

    public int? Update(Rect windowRect, int selected) {
        UpdateSelected(selected);
        HandleHover(Event.current.mousePosition, windowRect);


        int? nextSelected = null;

        switch (Event.current.type) {
            case EventType.MouseUp:
                if (Event.current.button == 0) {
                    nextSelected = HandleClick(Event.current.mousePosition, windowRect);
                }

                break;

            case EventType.MouseDrag:
                if (Event.current.button == 2 ||
                    (Event.current.modifiers & EventModifiers.Alt) == EventModifiers.Alt
                   ) {
                    HandleDrag(
                        Event.current.delta,
                        Event.current.mousePosition,
                        windowRect
                    );
                }

                break;

            case EventType.ScrollWheel:
                HandleScroll(
                    Event.current.mousePosition,
                    Event.current.delta.y,
                    windowRect
                );
                break;
        }

        if (_firstDraw) {
            ResetCamera(windowRect.size);
            _firstDraw = false;
        }

        _previewScene.BeginPreview(windowRect, "window");
        _previewScene.camera.Render();
        _previewScene.EndAndDrawPreview(windowRect);

        return nextSelected;
    }

    private Vector2 GetRecommendedSize(Vector2 size) {
        var wantWidth = _objectsPerRow * CellWorldSize;

        // We assume here that the camera is not rotated about the Y or Z axes.
        var aspect = size.y / size.x;
        var wantHeight = wantWidth * aspect;

        return new Vector2(wantWidth, wantHeight);
    }

    // Re-position the camera so that the first object is in the upper-left
    // corner, and re-zoom the camera so that a single row fits exactly in the
    // width of the palette.
    private void ResetCamera(Vector2 size) {
        var wantSize = GetRecommendedSize(size);

        var cam = _previewScene.camera;

        // The camera's orthosize is equal to half of the viewport's height in
        // worldspace.
        cam.orthographicSize = wantSize.y / 2;

        // The starting position on the Z axis is slightly tricky, since the
        // camera is tilted around the X axis, and the objects themselves are
        // scaled down and then rotated around the Y axis. For simplicity, let's
        // just assume that we want the top-north-west corner of the first cell
        // to be in view.
        //
        // To position the camera, we'll use the following technique: we'll
        // start with a camera position that is projected a sufficient distance
        // away from top-north-west corner at the angle provided by the camera's
        // X tilt. The distance is arbitrary as long as our objects end up
        // between our near/far planes.
        //
        // This will provide us with the worldspace position of the upper-left
        // corner of the ortho viewport. Since we calculated the worldspace size
        // of the viewport above, we can derive the camera's worldspace
        // position.
        const float halfCell = CellWorldSize / 2;
        var topNorthWestWorldPos = new Vector3(-halfCell, halfCell, halfCell);
        var toCam = Quaternion.Euler(CameraXRot, 0, 0) * Vector3.back;
        var ray = new Ray(topNorthWestWorldPos, toCam);
        var viewUpperLeftWorldPos = ray.GetPoint(10);
        var camTopToBottomDir = Quaternion.Euler(CameraXRot, 0, 0) * Vector3.down;
        var centerOffset = cam.orthographicSize * camTopToBottomDir +
                           0.5f * wantSize.x * Vector3.right;
        cam.transform.position = viewUpperLeftWorldPos + centerOffset;
    }

    private Vector3? GetScenePosFromMousePos(Vector2 mousePos, Rect windowRect) {
        var cameraOriginWindowSpace = new Vector2(
            windowRect.min.x,
            windowRect.max.y
        );

        var mousePosOffset = mousePos - cameraOriginWindowSpace;
        var viewportPos = mousePosOffset / windowRect.size;
        viewportPos.y *= -1;

        var ray = _previewScene.camera.ViewportPointToRay(viewportPos);
        var plane = new Plane(Vector3.up, Vector3.zero);
        if (!plane.Raycast(ray, out var dist)) {
            return null;
        }

        return ray.GetPoint(dist);
    }

    private int? GetEntryByMousePos(Vector2 mousePos, Rect windowRect) {
        var maybeWorldPos = GetScenePosFromMousePos(mousePos, windowRect);
        if (!maybeWorldPos.HasValue) {
            return null;
        }

        var origin = new Vector3(
            -0.5f * CellWorldSize,
            0,
            0.5f * CellWorldSize
        );

        var worldPos = maybeWorldPos.Value;
        if (worldPos.z > origin.z) {
            return null;
        }

        var maxX = origin.x + _objectsPerRow * CellWorldSize;
        if (worldPos.x < origin.x || maxX < worldPos.x) {
            return null;
        }

        var offset = worldPos - origin;
        var row = Mathf.FloorToInt(Mathf.Abs(offset.z));
        var col = Mathf.FloorToInt(Mathf.Abs(offset.x));
        var item = row * _objectsPerRow + col;
        if (item < 0 || _numObjects <= item) {
            return null;
        }

        return item;
    }

    private Vector3 GetItemPos(int item) {
        var row = item / _objectsPerRow;
        var col = item % _objectsPerRow;
        return new Vector3(
            col * CellWorldSize,
            ObjectYPos,
            -(row * CellWorldSize)
        );
    }

    private void ShowHover(int item) {
        var pos = GetItemPos(item);
        pos.y = 0;
        _blockEditorPaletteSelector.hoverDisplay.SetActive(true);
        _blockEditorPaletteSelector.hoverDisplay.transform.position = pos;
    }

    private void HideHover() {
        _blockEditorPaletteSelector.hoverDisplay.SetActive(false);
    }

    private void UpdateSelected(int item) {
        if (item < 0) {
            _blockEditorPaletteSelector.selectedDisplay.SetActive(false);
            return;
        }

        _blockEditorPaletteSelector.selectedDisplay.SetActive(true);

        var pos = GetItemPos(item);
        pos.y = 0;
        _blockEditorPaletteSelector.selectedDisplay.transform.position = pos;
    }

    private void HandleHover(Vector2 mousePos, Rect windowRect) {
        var item = GetEntryByMousePos(mousePos, windowRect);
        if (!item.HasValue) {
            HideHover();
            return;
        }

        ShowHover(item.Value);
    }

    private int? HandleClick(Vector2 mousePos, Rect windowRect) {
        var item = GetEntryByMousePos(mousePos, windowRect);
        if (!item.HasValue) {
            return null;
        }

        UpdateSelected(item.Value);
        return item.Value;
    }

    private void HandleDrag(Vector2 mouseDelta, Vector2 mouseCurPos, Rect windowRect) {
        var mousePrevPos = mouseCurPos - mouseDelta;
        if (!windowRect.Contains(mousePrevPos)) {
            return;
        }

        const float dragSpeed = 0.015f;
        var dragDelta = dragSpeed * mouseDelta;
        _previewScene.camera.transform.position += new Vector3(
            -dragDelta.x,
            0,
            dragDelta.y
        );
    }

    private void HandleScroll(Vector2 mousePos, float amount, Rect windowRect) {
        if (!windowRect.Contains(mousePos)) {
            return;
        }

        // The camera's orthosize is equal to half of the viewport's height in worldspace.
        var recZoom = GetRecommendedSize(windowRect.size).y / 2;

        const float scrollSpeed = 0.075f;
        _previewScene.camera.orthographicSize = Mathf.Clamp(
            _previewScene.camera.orthographicSize + scrollSpeed * amount,
            recZoom / 2,
            recZoom * 5
        );
    }

    public void Cleanup() {
        _previewScene.Cleanup();
    }
}

Can't upload images directly to the gist itself, but you can upload to comments!