Scene3D

Scene3D.java

    public static void onMove(int pointer, Ray ray) {
        Pointer p = pointers.get(pointer);
        if (p == null) {
            p = new Pointer();
            p.id = pointer;
            pointers.put(pointer, p);
        }

        for (UIStage stage : stages) {
            float px = -1;
            float py = -1;

            Matrix4 transform = stage.getInstance().transform;

            Vector3 a = tmp1.set(-stage.width / 2, 0, -stage.height / 2).mul(transform);
            Vector3 b = tmp2.set(stage.width / 2, 0, -stage.height / 2).mul(transform);
            Vector3 c = tmp3.set(stage.width / 2, 0, stage.height / 2).mul(transform);
            Vector3 d = tmp4.set(-stage.width / 2, 0, stage.height / 2).mul(transform);

            Vector3 intersection = tmp5;
            if (Intersector.intersectRayTriangle(ray, a, b, c, intersection)) {
                // First half
                // From intersection we calculate the barycenter
                // From the barycenter we can calculate the localspace
                Vector3 weight = barycentric(a, b, c, intersection, new Vector3());
                // A = (0, 0)
                // B = (1, 0)
                // C = (1, 1)
                float tx = 0 * weight.x + 1 * weight.y + 1 * weight.z;
                float tz = 0 * weight.x + 0 * weight.y + 1 * weight.z;

                px = stage.getTextureWidth() * tx;
                py = stage.getTextureHeight() * tz;

            } else if (Intersector.intersectRayTriangle(ray, a, c, d, intersection)) {
                Vector3 weight = barycentric(a, c, d, intersection, new Vector3());
                // A = (0, 0)
                // C = (1, 1)
                // D = (0, 1)
                float tx = 0 * weight.x + 1 * weight.y + 0 * weight.z;
                float tz = 0 * weight.x + 1 * weight.y + 1 * weight.z;

                px = stage.getTextureWidth() * tx;
                py = stage.getTextureHeight() * tz;
            } else {
                p.x = (int) px;
                p.y = (int) py;

                stage.dirty = true;
                stage.stage().mouseMoved(p.x, p.y);
                stage.stage().act();

                p.lastStage = null;
                continue;
            }

            if (p.x != (int) px || p.y != (int) py) {
                p.x = (int) px;
                p.y = (int) py;
                p.lastStage = stage;
                
                stage.dirty = true;
                stage.stage().mouseMoved(p.x, p.y);
                stage.stage().act();
                break;
            }
        }
    }

    private static final Vector3 v0 = new Vector3();
    private static final Vector3 v1 = new Vector3();
    private static final Vector3 v2 = new Vector3();

    // Thanks mgsx!
    // https://github.com/mgsx-dev/dl13/blob/8afbbf8e161320a47b6184c1b7ece47cf3cb1e61/core/src/net/mgsx/dl13/navmesh/NavMesh.java#L136
    public static Vector3 barycentric(Vector3 a, Vector3 b, Vector3 c, Vector3 p, Vector3 result) {
        v0.set(b).sub(a);
        v1.set(c).sub(a);
        v2.set(p).sub(a);

        float d00 = v0.dot(v0);
        float d01 = v0.dot(v1);
        float d11 = v1.dot(v1);
        float d20 = v2.dot(v0);
        float d21 = v2.dot(v1);
        float denom = d00 * d11 - d01 * d01;
        if (denom == 0) {
            denom = 1;
        }
        float v = (d11 * d20 - d01 * d21) / denom;
        float w = (d00 * d21 - d01 * d20) / denom;
        float u = 1.0f - v - w;

        return result.set(u, v, w);
    }
    
    static class Pointer {
        boolean pressed;
        int x, y, id;
        UIStage lastStage;
    }

Please note: The code expects your original plane to be in the XZ plane before applying transforms.