Implementing a 1-bit hue-based palette shader in the ReShade FX language.

hbpg.fx

#include "ReShade.fxh"

/*
1-bit Hue-Based Palette Shader
Created by Expack3, adapting source code from the following sources:
"Dithering on the GPU" by Alex Charlton: http://alex-charlton.com/posts/Dithering_on_the_GPU
"RGB to HSV/HSL/HCY/HCL in HLSL" by Ian Taylor (Copyright © 2002-2018 Ian Taylor): http://www.chilliant.com/rgb2hsv.html

Thanks to EDCVBNM and Matsilagi for their invaluable assistance!
*/

#define Epsilon 0.0000000001
static int indexMatrix4x4[16] = {0,  8,  2,  10,
                                     12, 4,  14, 6,
                                     3,  11, 1,  9,
                                     15, 7,  13, 5};

sampler2D SourcePointSampler
{
	Texture = ReShade::BackBufferTex;
	MinFilter = POINT;
	MagFilter = POINT;
	MipFilter = POINT;
	AddressU = CLAMP;
	AddressV = CLAMP;
};

float mod(float x, float y) 
    {
        return x - y * floor(x / y);
    }

float3 HUEtoRGB(float H)
  {
    float R = abs(H * 6.0 - 3.0) - 1.0;
    float G = 2.0 - abs(H * 6.0 - 2.0);
    float B = 2.0 - abs(H * 6.0 - 4.0);
    return clamp(float3(R,G,B),0,1);
  }

float3 HSLtoRGB(float3 HSL)
{
  float3 RGB = HUEtoRGB(HSL.x);
  float C = (1.0 - abs(2 * HSL.z - 1)) * HSL.y;
  return (RGB - 0.5) * C + HSL.z;
}

float3 RGBtoHCV(float3 RGB)
{
  // Based on work by Sam Hocevar and Emil Persson
  float4 P = (RGB.g < RGB.b) ? float4(RGB.bg, -1.0, 2.0/3.0) : float4(RGB.gb, 0.0, -1.0/3.0);
  float4 Q = (RGB.r < P.x) ? float4(P.xyw, RGB.r) : float4(RGB.r, P.yzx);
  float C = Q.x - min(Q.w, Q.y);
  float H = abs((Q.w - Q.y) / (6.0 * C + Epsilon) + Q.z);
  return float3(H, C, Q.x);
}

float3 RGBtoHSV(float3 RGB)
  {
    float3 HCV = RGBtoHCV(RGB);
    float S = HCV.y / (HCV.z + Epsilon);
    return float3(HCV.x, S, HCV.z);
  }

float3 RGBtoHSL(float3 RGB)
  {
    float3 HCV = RGBtoHCV(RGB);
    float L = HCV.z - HCV.y * 0.5;
    float S = HCV.y / (1.0 - abs(L * 2.0 - 1.0) + Epsilon);
    return float3(HCV.x, S, L);
  }

float lightnessStep(float l) {
    // Quantize the lightness to one of `lightnessSteps` values
    return floor(0.5 + l);
}

float hueDistance(float h1, float h2) {
    float diff = abs((h1 - h2));
    return min(abs((1.0 - diff)), diff);
}

//while this method requires SV_Position as input, it only needs its X and Y components
float indexValue(float2 texloc) {
    int x = int(mod(texloc.x, 4));
    int y = int(mod(texloc.y, 4));
    return indexMatrix4x4[(x + y * 4)] / 16.0;
}

void closestColors(float hue, out float3 ret1, out float3 ret2) {
    float3 closest = float3(-2, 0, 0);
    float3 secondClosest = float3(-2, 0, 0);
    float3 temp;

    //unrolled for loop since pallete is always 1-bit BW

    //i==0
    temp = float3(0,0,0);
    float tempDistance = hueDistance(temp.x, hue);
    if (tempDistance < hueDistance(closest.x, hue)) {
        secondClosest = closest;
        closest = temp;
    } else {
        if (tempDistance < hueDistance(secondClosest.x, hue)) {
            secondClosest = temp;
        }
    }
    ret1 = closest;
    ret2 = secondClosest;

    //i==1
    temp = float3(255,255,255);
    tempDistance = hueDistance(temp.x, hue);
    if (tempDistance < hueDistance(closest.x, hue)) {
        secondClosest = closest;
        closest = temp;
    } else {
        if (tempDistance < hueDistance(secondClosest.x, hue)) {
            secondClosest = temp;
        }
    }
    ret1 = closest;
    ret2 = secondClosest;
}

float3 dither(float3 color, float2 texlocation) {
    float3 hsl = RGBtoHSL(color);

    float3 cs1, cs2;

    closestColors(hsl.x, cs1, cs2);
    float3 c1 = cs1;
    float3 c2 = cs2;
    float d = indexValue(texlocation);
    float hueDiff = hueDistance(hsl.x, c1.x) / hueDistance(c2.x, c1.x);

    float l1 = lightnessStep(max((hsl.z - 0.125), 0.0));
    float l2 = lightnessStep(min((hsl.z + 0.124), 1.0));
    float lightnessDiff = (hsl.z - l1) / (l2 - l1);

    float3 resultColor = (hueDiff < d) ? c1 : c2;
    resultColor.z = (lightnessDiff < d) ? l1 : l2;
    return HSLtoRGB(resultColor);
}

float4 PS_HPBS(float4 vpos : SV_Position, float2 texcoord : TEXCOORD) : SV_Target
{
	float3 fragcolor = tex2D(SourcePointSampler, texcoord).rgb;
	return float4(dither(fragcolor, vpos.xy), 1.0);
}


technique HPBG

{

	pass PAL

	{

		VertexShader = PostProcessVS;

		PixelShader = PS_HPBS;

	}

}