Quaternion (un)packing function, smallest-three representation.

pack_quaternion.hpp

#pragma once

#include <glm/gtc/quaternion.hpp>

#include <cassert>
#include <cmath>
#include <cstdint>

/** Creates smallest-three packing of quaternion, 10 bits per component */
inline auto pack_quaternion(glm::quat q) -> std::uint32_t {
    constexpr auto quantize_max = 0b11'1111'1111u;

    // [0..1]: index of largest
    // [2..11]: first smallest
    // [12..21]: second smallest
    // [22..31]: third smallest
    auto pack = std::uint32_t{};

    auto is_biggest = [&](int i) -> bool {
        auto x = std::abs(q[i]);

        for (int j = 0; j < 4; ++j) {
            if (j == i) {
                continue;
            }

            auto y = std::abs(q[j]);
            if (x < y) {
                return false;
            }
        }

        return true;
    };

    auto biggest = ([&] {
        for (int i = 0; i < 4; ++i) {
            if (is_biggest(i)) {
                return i;
            }
        }

        return 0;
    })();

    if (q[biggest] < 0) {
        q = -q;
    }

    pack |= biggest;

    for (auto i = 0; i < 3; ++i) {
        auto e = q[i >= biggest ? i + 1 : i];
        assert(e <= q[biggest]);

        auto quantized = static_cast<std::uint32_t>((e + 1.f) / 2.f * quantize_max);
        assert((quantized & quantize_max) == quantized);

        pack |= quantized << (2 + 10 * i);
        assert((pack >> (2 + 10 * i)) == quantized);
    }

    return pack;
}

/** Unpacks quaternion from smallest-three packing, 10 bits per component */
inline auto unpack_quaternion(std::uint32_t p) -> glm::quat {
    constexpr auto quantize_max = 0b11'1111'1111u;

    auto quat = glm::quat{};

    auto biggest = p & 0b11;

    auto acc = 0.f;

    for (auto i = 0; i < 3; ++i) {
        auto quantized = (p >> (2 + 10 * i)) & quantize_max;
        auto v = float(quantized) / float(quantize_max) * 2.f - 1.f;
        assert(v >= -1.f && v <= 1.f);

        acc += v * v;

        quat[i >= biggest ? i + 1 : i] = v;
    }

    quat[biggest] = std::sqrt(1.f - acc);
    assert(quat[biggest] >= -1.f && quat[biggest] <= 1.f);

    return glm::normalize(quat);
}