Multisampled SDF for text rendering (4x AA with single texture lookup, 16x with four lookups)

multisampled_font_sdf.glsl

// Multisampled SDF for text rendering (4x AA with single texture lookup, 16x with four lookups)
// It is for SDF text rendering with 16x antialiasing with only four texture lookups, where an 
// SDF texture texel in turn also has four individual SDF values in the RGBA color channels in 
// a 4-Rook/RGSS sampling pattern.

// Copyright (C) 2022, Benjamin 'BeRo' Rosseaux - License: Unlicense ( http://unlicense.org/ )

const float SQRT_0_DOT_5 = sqrt(0.5);

#if FILLTYPE == FILLTYPE_ATLAS_TEXTURE
  #define ADJUST_TEXCOORD(uv) vec3(uv, texCoord.z)
  #define TVEC vec3
#else
  #define ADJUST_TEXCOORD(uv) uv
  #define TVEC vec2
#endif

#if ((FILLTYPE == FILLTYPE_TEXTURE) || (FILLTYPE == FILLTYPE_ATLAS_TEXTURE))

// 4x multisampled 4-rook/RGSS SDF with a single texture lookup of four SDF values in the RGBA color channels 
float sampleSDF(const in TVEC texCoord){
  const float HALF_BY_SQRT_TWO = 0.5 / sqrt(2.0), 
              ONE_BY_THREE = 1.0 / 3.0, 
              NORMALIZATION_THICKNESS_SCALE = SQRT_0_DOT_5 * (0.5 / 4.0);     
  vec4 texel = textureLod(uTexture, texCoord, 0.0);
  vec4 gradientX = dFdx(texel);
  vec4 gradientY = dFdy(texel);
  vec2 gradients[4] = vec2[4](
    vec2(gradientX.x, gradientY.x),
    vec2(gradientX.y, gradientY.y),
    vec2(gradientX.z, gradientY.z),
    vec2(gradientX.w, gradientY.w)
  );
  vec4 gradientSquaredLengths = vec4(dot(gradients[0], gradients[0]), 
                                     dot(gradients[1], gradients[1]), 
                                     dot(gradients[2], gradients[2]), 
                                     dot(gradients[3], gradients[3]));
  gradients[0] = (gradientSquaredLengths[0] < 1e-4) ? vec2(SQRT_0_DOT_5) : (gradients[0] * inversesqrt(max(gradientSquaredLengths[0], 1e-4)));
  gradients[1] = (gradientSquaredLengths[1] < 1e-4) ? vec2(SQRT_0_DOT_5) : (gradients[1] * inversesqrt(max(gradientSquaredLengths[1], 1e-4)));
  gradients[2] = (gradientSquaredLengths[2] < 1e-4) ? vec2(SQRT_0_DOT_5) : (gradients[2] * inversesqrt(max(gradientSquaredLengths[2], 1e-4)));
  gradients[3] = (gradientSquaredLengths[3] < 1e-4) ? vec2(SQRT_0_DOT_5) : (gradients[3] * inversesqrt(max(gradientSquaredLengths[3], 1e-4)));
  vec2 texTexelCoord = texCoord.xy * textureSize(uTexture, 0).xy;
  vec2 Juv[4] = vec2[4](
    vec2(texTexelCoord + vec2(0.125, 0.375)),
    vec2(texTexelCoord + vec2(-0.125, -0.375)),
    vec2(texTexelCoord + vec2(0.375, -0.125)),
    vec2(texTexelCoord + vec2(-0.375, 0.125))
  );
  vec4 Jdxdy[4] = vec4[4](
    vec4(dFdx(Juv[0]), dFdy(Juv[0])),
    vec4(dFdx(Juv[1]), dFdy(Juv[1])),
    vec4(dFdx(Juv[2]), dFdy(Juv[2])),
    vec4(dFdx(Juv[3]), dFdy(Juv[3]))
  );
  vec2 jacobianGradients[4] = vec2[4](
#if 0
    vec2(mat2(gradients[0], gradients[0]) * mat2(Jdxdy[0].xy, Jdxdy[0].zw)),
    vec2(mat2(gradients[1], gradients[1]) * mat2(Jdxdy[1].xy, Jdxdy[1].zw)),
    vec2(mat2(gradients[2], gradients[2]) * mat2(Jdxdy[2].xy, Jdxdy[2].zw)),
    vec2(mat2(gradients[3], gradients[3]) * mat2(Jdxdy[3].xy, Jdxdy[3].zw))
#else
    vec2((gradients[0].x * Jdxdy[0].x) + (gradients[0].y * Jdxdy[0].z), (gradients[0].x * Jdxdy[0].y) + (gradients[0].y * Jdxdy[0].w)),
    vec2((gradients[1].x * Jdxdy[1].x) + (gradients[1].y * Jdxdy[1].z), (gradients[1].x * Jdxdy[1].y) + (gradients[1].y * Jdxdy[1].w)),
    vec2((gradients[2].x * Jdxdy[2].x) + (gradients[2].y * Jdxdy[2].z), (gradients[2].x * Jdxdy[2].y) + (gradients[2].y * Jdxdy[2].w)),
    vec2((gradients[3].x * Jdxdy[3].x) + (gradients[3].y * Jdxdy[3].z), (gradients[3].x * Jdxdy[3].y) + (gradients[3].y * Jdxdy[3].w))
#endif
  );
  vec4 widths = min(vec4(length(jacobianGradients[0]), 
                         length(jacobianGradients[1]), 
                         length(jacobianGradients[2]), 
                         length(jacobianGradients[3])) * NORMALIZATION_THICKNESS_SCALE, vec4(0.5));
  return dot(linearstep(vec4(0.5) - widths, vec4(0.5) + widths, texel), vec4(0.25)); 
}

// In the best case effectively 16x multisampled SDF, otherwise just 4x in the worst case, depending on the texCoord gradient derivatives 
float multiSampleSDF(const in TVEC texCoord){
  const float HALF_BY_SQRT_TWO = 0.5 / sqrt(2.0);
  vec4 buv = texCoord.xyxy + (vec2((dFdx(texCoord.xy) + dFdy(texCoord.xy)) * HALF_BY_SQRT_TWO).xyxy * vec2(-1.0, 1.0).xxyy);
  return dot(vec4(sampleSDF(ADJUST_TEXCOORD(buv.xy)), sampleSDF(ADJUST_TEXCOORD(buv.zy)), sampleSDF(ADJUST_TEXCOORD(buv.xw)), sampleSDF(ADJUST_TEXCOORD(buv.zw))), vec4(0.25));
}
#endif