Fragment Shader to test color-keyed transparency in picom

sample-shader_color-key.frag

#version 330
uniform float opacity;
uniform float dim;
uniform bool invert_color;
in vec2 texcoord;
uniform sampler2D tex;
uniform sampler2D brightness;
uniform float max_brightness;

float get_color_distance(vec3 a, vec3 b) {
    float r_mean = (a.r + b.r) / 2.0;
    vec3 c_diff = a - b;
    float c_dist = dot(vec3(2 + r_mean, 4, 3 - r_mean), c_diff * c_diff) / 3.0;
    c_dist = min(c_dist * 300, 1.0);
    return pow(c_dist, 2);
}

float get_bg_alpha(vec3 bg_color, float bg_opacity) {
    float dist = get_color_distance(bg_color.rgb, texelFetch(tex, ivec2(texcoord), 0).rgb) * 4.0;
    dist += get_color_distance(bg_color.rgb, texelFetch(tex, ivec2(texcoord + vec2(0, 1)), 0).rgb) * 2.0;
    dist += get_color_distance(bg_color.rgb, texelFetch(tex, ivec2(texcoord + vec2(1, 1)), 0).rgb) * 1.0;
    dist += get_color_distance(bg_color.rgb, texelFetch(tex, ivec2(texcoord + vec2(1, 0)), 0).rgb) * 2.0;
    dist += get_color_distance(bg_color.rgb, texelFetch(tex, ivec2(texcoord + vec2(1, -1)), 0).rgb) * 1.0;
    dist += get_color_distance(bg_color.rgb, texelFetch(tex, ivec2(texcoord + vec2(0, -1)), 0).rgb) * 2.0;
    dist += get_color_distance(bg_color.rgb, texelFetch(tex, ivec2(texcoord + vec2(-1, -1)), 0).rgb) * 1.0;
    dist += get_color_distance(bg_color.rgb, texelFetch(tex, ivec2(texcoord + vec2(-1, 0)), 0).rgb) * 2.0;
    dist += get_color_distance(bg_color.rgb, texelFetch(tex, ivec2(texcoord + vec2(-1, 1)), 0).rgb) * 1.0;
    return mix(bg_opacity, 1, dist / 16.0);
}

void main() {
    vec3 bg_color = vec3(0.25, 0.247, 0.252);
    float bg_opacity = 0.5;

    vec4 c = texelFetch(tex, ivec2(texcoord), 0);
    float bg_alpha = get_bg_alpha(bg_color, bg_opacity);
    gl_FragColor = c * bg_alpha;
}