SNES Palette Aseprite script

snes-palette.lua

---------------------------------------------------------
-- Convert image to use SNES palette
-- (c) 2022 oldmanoftheSEA, CC-BY-SA 4.0
--
-- This work is licensed under the Creative Commons
-- Attribution-ShareAlike 4.0 International License. 
-- To view a copy of this license, visit
-- http://creativecommons.org/licenses/by-sa/4.0/.
---------------------------------------------------------

-- Forward declarations
local convertPalettes, convertRGBImage
local cachedConverter, convertColor, truncatingConverter, uniformConverter
local colorComparator, rgbToXyz, xyzToCIELab, deltaE94, distance

local function run()
    local sprite = {sprite=app.activeSprite, frame=app.activeFrame}
    local cvt = uniformConverter -- or truncatingConverter
    if sprite.sprite.colorMode == ColorMode.INDEXED then
        app.transaction(function() convertPalettes(cvt, sprite) end)
    elseif sprite.sprite.colorMode == ColorMode.RGB then
        app.transaction(function() convertRGBImage(cvt, sprite) end)
    else
        app.alert("Unsupported color mode")
    end
end

function convertPalettes(cvt, sprite)
    local converter = cachedConverter(cvt)
    for i = 1, #sprite.sprite.palettes do		-- NOTE: aseprite bug (ipairs on this produces an error)
        local pal = sprite.sprite.palettes[i]
        for j = 0, #pal-1 do
            pal:setColor(j, converter(pal:getColor(j)))
        end
    end
end

function convertRGBImage(cvt, sprite)
    local converter = cachedConverter(cvt)
    for i = #sprite.sprite.cels, 1, -1 do
        local cel = sprite.sprite.cels[i]
        local img = cel.image:clone()
        for it in img:pixels() do
            local c = Color(it())
            if c.alpha > 0 then
                it(converter(c).rgbaPixel)
            end
        end
        cel.image = img
    end
end

function cachedConverter(cvt)
    local cache = {}
    local cvt = cvt
    return function(c)
        local dst = cache[c.rgbaPixel]
        if dst == nil then
            dst = convertColor(cvt, c)
            cache[c.rgbaPixel] = dst
        end
        return dst
    end
end

function convertColor(cvt, rgba)
    local c = Color(rgba)
    if c.alpha == 0 then
        return rgba
    end

    local cmp = colorComparator(rgba)
    local best = nil
    local bestdiff = 999999

    local rvalues = cvt.eightToFive(c.red)
    local gvalues = cvt.eightToFive(c.green)
    local bvalues = cvt.eightToFive(c.blue)
    for _, r in ipairs(rvalues) do
        for _, g in ipairs(gvalues) do
            for _, b in ipairs(bvalues) do
                local candidate = Color{r=cvt.fiveToEight(r), g=cvt.fiveToEight(g), b=cvt.fiveToEight(b)}
                local diff = cmp(candidate)
                if diff < bestdiff then
                    best = candidate
                    bestdiff = diff
                end
            end
        end
    end

    return best
end

truncatingConverter = {
    eightToFive = function(v)
        local hi = v >> 3
        if (v & 7) > 0 and hi < 31 then
            return {hi, hi + 1}
        else
            return {hi}
        end
    end,
    fiveToEight = function(v)
        return v << 3
    end,
}

uniformConverter = {
    eightToFive = function(v)
        local hi = v >> 3
        local lo = hi >> 2
        if (v & 7) > lo then
            return {hi, hi + 1}
        elseif (v & 7) < lo then
            return {hi - 1, hi}
        else
            return {hi}
        end
    end,
    fiveToEight = function(v)
        return (v << 3) + (v >> 2)
    end,
}

--------------------------------
-- Color math
-- a/k/a "Thanks, easyRGB.com!"
--------------------------------

-- Constant parameters to color math functions
local E94Weights = {K1=0.045, K2=0.015, kL=1.0, kC=1.0, kH=1.0}
local Illuminants = {
    D65 = {X=95.047, Y=100.0, Z=108.883},
}

-- Returns a function that differences with the provided color.
-- This pattern caches the CIELab conversion
function colorComparator(rgba)
    local cie = xyzToCIELab(rgbToXyz(rgba), Illuminants.D65)
    return function(other)
        local ret = deltaE94(cie, xyzToCIELab(rgbToXyz(other), Illuminants.D65), E94Weights)
        return ret
    end
end

-- Converts aseprite-native sRGB colors to the CIE 1931 "XYZ" color space
function rgbToXyz(rgba)
    local function scaleComponent(p)
        if p > 0.04045 then
            return ((p + 0.055) / 1.055) ^ 2.4
        else
            return p / 12.92
        end
    end

    local c = Color(rgba)
    local r = scaleComponent(c.red / 255.0) * 100.0
    local g = scaleComponent(c.green / 255.0) * 100.0
    local b = scaleComponent(c.blue / 255.0) * 100.0

    return {
        X = r * 0.4124 + g * 0.3576 + b * 0.1805,
        Y = r * 0.2126 + g * 0.7152 + b * 0.0722,
        Z = r * 0.0193 + g * 0.1192 + b * 0.9505,
    }
end

-- Converts XYZ colors to the CIEL*a*b color space
function xyzToCIELab(xyz, reference)
    local function scaleComponent(p)
        if p > 0.008856 then
            return p ^ (1.0 / 3.0)
        else
            return (7.787 * p) + (16.0 / 116.0)
        end
    end

    local x = scaleComponent(xyz.X / reference.X)
    local y = scaleComponent(xyz.Y / reference.Y)
    local z = scaleComponent(xyz.Z / reference.Z)

    return {
        L = (116.0 * y) - 16.0,
        a = 500.0 * (x - y),
        b = 200.0 * (y - z),
    }
end

-- CIE94 delta-E color difference
-- NOTE: There are newer versions but they carry much greater complexity and debatable upside.
function deltaE94(cie1, cie2, weights)
    local xC1 = distance(cie1.a, cie1.b)
    local xC2 = distance(cie2.a, cie2.b)
    local xDL = cie2.L - cie1.L
    local xDC = xC2 - xC1
    local xDE = distance(cie1.L - cie2.L, cie1.a - cie2.a, cie1.b - cie2.b)
    local xDH = math.sqrt(math.max(0, (xDE * xDE) - (xDL * xDL) - (xDC * xDC)))
    local xSC = 1 + (weights.K1 * xC1)
    local xSH = 1 + (weights.K2 * xC1)
    return distance(xDL / weights.kL, xDC / (weights.kC * xSC), xDH / (weights.kH * xSH))
end

-- Euclidean distance
function distance(...)
    local sumOfSquares = 0
    for i = 1, select("#", ...) do
        local v = select(i, ...)
        sumOfSquares = sumOfSquares + (v * v)
    end
    return math.sqrt(sumOfSquares)
end

run()