BadAA, an edge-detection- and edge-following-based post-processing AA shader for OpenMW

BadAAColorsVersion.omwfx

uniform_float uRange {
    default = 5;
    min = 1;
    max = 16.0;
    step = 1;
    description = "How many pixels the algorithm follows to try to detect an edge.";
    display_name = "Range";
}
uniform_float uThreshold {
    default = 20.0;
    min = 2.0;
    max = 100.0;
    step = 1;
    description = "Depth difference threshold between two pixels to treat that pixel pair as an edge. Low values cause more edges to be anti-aliased.";
    display_name = "Depth Threshold";
}
uniform_float uNormalDotOffset {
    default = 0.95;
    min = 0.0;
    max = 1.02;
    step = 0.01;
    description = "Factor to use when comparing surface angles to see if they're similar. Higher values make more surfaces be treated as different. 0.0 disables normals-based edge detection. Values above 1.0 are likely to cause serious problems.";
    display_name = "Normal Comparison Offset";
}
uniform_float uLumaLikenessThreshold {
    default = 0.15;
    min = 0.0;
    max = 1.0;
    step = 0.01;
    description = "Threshold to use when comparing color values. Low value cause the image to become indiscriminately blurry. Very high values disable color comparisons entirely, harming filter quality.";
    display_name = "Color Likeness Threshold";
}
uniform_bool uDoDiagonals {
    default = true;
    description = "Whether to attempt to process diagonals or not. If disabled, treats diagonals like horizontal/vertical edges. They will still be softened, but only for one or two pixels.";
    display_name = "Process Diagonals";
}
uniform_bool uDepthSqrtHack {
    default = true;
    description = "Compare with square root depth instead of true depth. Makes faraway rocks and hills less distorted.";
    display_name = "Use Depth Sqrt Hack";
}
uniform_bool uDepthNormalHack {
    default = true;
    description = "Modify depth threshold based on surface normal. Makes faraway rocks and hills less distorted.";
    display_name = "Use Depth Normal Hack";
}
uniform_float uWaterHack {
    default = 4.0;
    min = 0.0;
    max = 16.0;
    step = 0.1;
    description = "Controls the range of the hack that encourages water to be treated differently from non-water. This uses normals, so Normal Comparison Offset must be nonzero for it to work.";
    display_name = "Water Hack Range";
}
uniform_bool uSkipAll {
    default = false;
    description = "Skip all processing. For debugging.";
    display_name = "Skip All Processing";
}
uniform_int uOutputDebug {
    default = 0;
    min = 0;
    max = 16;
    description = "Indescribable debugging thing.";
    display_name = "Debugging Thing";
}
uniform_float uGamma {
    default = 2.2;
    min = 0.1;
    max = 4.0;
    step = 0.05;
    description = "Gamma compression to use when blending colors. 2.2 is (approximately) physically correct and avoids darkening when blending very different highly saturated colors. Values close to 1.0 approach what people expect from image editors, but have the different-saturated-color-darkening problem.";
    display_name = "Gamma";
}
uniform_bool uMSAACompatibilityHack {
    default = false;
    description = "Turn off the anti-aliasing filter in areas of the image where it thinks that MSAA might be active.";
    display_name = "MSAA Compatibility Hack";
}

shared {
    vec4 powv(vec4 a, float x)
    {
        return pow(a, vec4(x));
    }
    mat3 getInfo(vec2 texcoord)
    {
        float depth = omw_GetLinearDepth(texcoord);
        vec3 normal = omw_GetNormals(texcoord).rgb;
        vec3 luma   = powv(omw_GetLastShader(texcoord), 1.0/uGamma).rgb;
        // water doesn't write to the depth buffer.
        // hack: detect if sampled point is on other side of water and override its depth if so
        if(omw.isWaterEnabled)
        {
            float eye_height = omw.eyePos.z;
            float world_height = omw_GetWorldPosFromUV(texcoord).z;
            if((eye_height >= omw.waterHeight && world_height < omw.waterHeight) || (eye_height < omw.waterHeight && world_height > omw.waterHeight))
            {
                if(uWaterHack > 0.0)
                {
                    // if close to shore: output a dummy normal too, to help the normal comparisons do their thing
                    float hack_depth = abs(world_height - omw.waterHeight)/(uWaterHack*depth/256.0);
                    // two-stage interpolation so that we don't get any seams
                    if(hack_depth > 0.5 && hack_depth < 1.0)
                        normal = normalize(mix(vec3(normal.x, normal.z, normal.y), normal, hack_depth*2.0-1.0));//vec3(0.0, 0.0, -1.0);
                    else if(hack_depth <= 0.5)
                        normal = normalize(mix(-normal, vec3(normal.x, normal.z, normal.y), hack_depth*2.0));//vec3(0.0, 0.0, -1.0);
                }
                // fix depth
                float depth_adjust = (omw.waterHeight - world_height) / (eye_height - world_height);
                depth = mix(depth, 0.0, depth_adjust);
            }
        }
        return mat3(normal.x, normal.y, normal.z, luma.r, luma.g, luma.b, depth, 0.0, 0.0);
    }
}

render_target RT_NoMipmap {
    internal_format = rgb16f;
    source_type = float;
    source_format = rgb;
    mipmaps = false;
    min_filter = nearest;
    mag_filter = nearest;
}

fragment nomipmap(target=RT_NoMipmap) {
    omw_In vec2 omw_TexCoord;

    void main()
    {
        // copy into a buffer with no filtering or mipmapping (where we're going, we don't need it)
        // do the gamma compression step while we're at it
        omw_FragColor = powv(omw_GetLastShader(omw_TexCoord), uGamma);
    }
}


fragment BAD(rt1=RT_NoMipmap) {
    #define NORMAL_LIKENESS_THRESHOLD 0.05
    #define NORMALS_DISABLED_VALUE 0.0
    omw_In vec2 omw_TexCoord;
    
    vec3 comparify(mat3 info_a, mat3 info_b)
    {
        vec3 normal_a = vec3(info_a[0][0], info_a[0][1], info_a[0][2]);
        vec3 normal_b = vec3(info_b[0][0], info_b[0][1], info_a[0][2]);
        float depth_diff = info_a[2][0] - info_b[2][0];
        float luma_diff = distance(vec3(info_a[1][0], info_a[1][1], info_a[1][2]), vec3(info_b[1][0], info_b[1][1], info_b[1][2]))/1.75;
        
        if(uDepthSqrtHack)
            depth_diff = sqrt(info_a[2][0]/1024.0)*1024.0 - sqrt(info_b[2][0]/1024.0)*1024.0;
        if(uDepthNormalHack) // might want to scale this hack by depth too (only make it happen at a distance)
        {
            float view_dot_a = dot(omw.eyeVec.xyz, normal_a);
            float view_dot_b = dot(omw.eyeVec.xyz, normal_b);
            float view_dot = min(view_dot_a, view_dot_b); // -1.0: camera facing towards object, 1.0: away
            view_dot = abs(view_dot); // negative values don't concern us; now 1.0 = camera facing towards/away, 0.0 = parallel with camera forward dir
            float factor = sqrt(info_a[2][0]/1024.0); // adjust adjustment relative to sqrt of depth
            depth_diff *= mix(1.5, 0.01, clamp(view_dot*factor, 0.0, 1.0));
        }
        
        depth_diff = abs(depth_diff);
        
        float dp = dot(normal_a, normal_b); // 1.0 to -1.0, 1.0 = same as info_b
        float normal_diff = 1.0 - dp; // now 0.0 to 2.0 (0.0 being more similar)
        normal_diff /= 1.0; // 0.0 to 1.0
        normal_diff += uNormalDotOffset*2.0;
        normal_diff = max(0.0, normal_diff-2.0);
        if(length(normal_a) > 1.1 || info_a[2][0] > omw.far*0.95) // bogus normal, probably a rendering bug, or the sky. treat as same as neighbor (i.e. fall back to depth)
            normal_diff = 0.0;
        
        return vec3(normal_diff, depth_diff, luma_diff);
    }
    
    bool comparison_similar(vec3 comparison, int mode)
    {
        bool depth_similar = comparison.y <= uThreshold;
        bool normal_similar = comparison.x < NORMAL_LIKENESS_THRESHOLD;
        bool luma_similar = comparison.z < uLumaLikenessThreshold;
        if(mode == 0)
            return depth_similar;
        else if(mode == 1)
            return normal_similar;
        else if(mode == 2)
            return luma_similar;
        else
            return depth_similar && normal_similar && luma_similar;
    }
    float is_similar(vec3 comparison, int mode)
    {
        return comparison_similar(comparison, mode) ? 1.0 : 0.0;
    }

    void main()
    {
        vec2 texcoord = omw_TexCoord;
        vec2 res = omw.rcpResolution.xy;
        
        if(uSkipAll)
        {
            omw_FragColor = powv(omw_Texture2D(RT_NoMipmap, texcoord), 1.0/uGamma);
            return;
        }
        
        mat2 to_px = mat2(
            res.x, 0.0,
            0.0, res.y
        );
        
        mat3 center = getInfo(texcoord);
        vec3 top    = comparify(getInfo(texcoord + vec2( 0.0, -1.0)*to_px), center);
        vec3 bottom = comparify(getInfo(texcoord + vec2( 0.0,  1.0)*to_px), center);
        vec3 left   = comparify(getInfo(texcoord + vec2(-1.0,  0.0)*to_px), center);
        vec3 right  = comparify(getInfo(texcoord + vec2( 1.0,  0.0)*to_px), center);
        
        bool no_depth_edge  = max(max(abs(top.y), abs(bottom.y)), max(abs(left.y), abs(right.y))) <= uThreshold;
        bool no_normal_edge = max(max(abs(top.x), abs(bottom.x)), max(abs(left.x), abs(right.x))) <= NORMAL_LIKENESS_THRESHOLD || uNormalDotOffset == NORMALS_DISABLED_VALUE;
        bool no_luma_edge   = max(max(abs(top.z), abs(bottom.z)), max(abs(left.z), abs(right.z))) <= uLumaLikenessThreshold;
        
        omw_FragColor = vec4(1.0);
        // skip if no edges
        if(no_depth_edge && no_normal_edge && no_luma_edge)
        {
            omw_FragColor = powv(omw_Texture2D(RT_NoMipmap, texcoord), 1.0/uGamma);
        }
        else
        {
            // edge AA
            
            int mode = no_depth_edge ? (no_normal_edge ? 2 : 1) : 0;
            float tl = 1.0 - is_similar(comparify(getInfo(texcoord + vec2(-1.0, -1.0)*to_px), center), mode);
            float tr = 1.0 - is_similar(comparify(getInfo(texcoord + vec2( 1.0, -1.0)*to_px), center), mode);
            float bl = 1.0 - is_similar(comparify(getInfo(texcoord + vec2(-1.0,  1.0)*to_px), center), mode);
            float br = 1.0 - is_similar(comparify(getInfo(texcoord + vec2( 1.0,  1.0)*to_px), center), mode);
            float t_scalar = 1.0 - is_similar(top   , mode);
            float b_scalar = 1.0 - is_similar(bottom, mode);
            float l_scalar = 1.0 - is_similar(left  , mode);
            float r_scalar = 1.0 - is_similar(right , mode);
            
            vec4 raw_base_color = omw_Texture2D(RT_NoMipmap, texcoord);
            vec4 base_color = raw_base_color;
            
            float t_compare  = true ? (tl*0.25 + tr*0.25 + t_scalar*0.5) : t_scalar;
            float b_compare  = true ? (bl*0.25 + br*0.25 + b_scalar*0.5) : b_scalar;
            float l_compare  = true ? (tl*0.25 + bl*0.25 + l_scalar*0.5) : l_scalar;
            float r_compare  = true ? (tr*0.25 + br*0.25 + r_scalar*0.5) : r_scalar;
            
            float tl_compare = uDoDiagonals ? 0.95*(t_scalar*0.5 + l_scalar*0.5) : 0.0;
            float tr_compare = uDoDiagonals ? 0.95*(t_scalar*0.5 + r_scalar*0.5) : 0.0;
            float bl_compare = uDoDiagonals ? 0.95*(b_scalar*0.5 + l_scalar*0.5) : 0.0;
            float br_compare = uDoDiagonals ? 0.95*(b_scalar*0.5 + r_scalar*0.5) : 0.0;
            
            // figure out what type of edge we're on and configure the edge search accordintly
            
            mat2 to_px_2 = mat2(0.0, 0.0, 0.0, 0.0);
            vec2 center_pos = vec2(0.0);
            float max_compare = max(max(max(t_compare, b_compare), max(l_compare, r_compare)), max(max(tl_compare, bl_compare), max(tr_compare, br_compare)));
            bool skip = false;
            // FIXME there's got to be a way to simplify this
            if(t_compare >= max_compare)
            {
                center_pos = vec2( 0.0, -0.5);
                to_px_2 = mat2(res.x, 0.0, 0.0, res.y);
            }
            else if(b_compare >= max_compare)
            {
                center_pos = vec2( 0.0,  0.5);
                to_px_2 = mat2(res.x, 0.0, 0.0, -res.y);
            }
            else if(l_compare >= max_compare)
            {
                center_pos = vec2(-0.5,  0.0);
                to_px_2 = mat2(0.0, res.x, res.y, 0.0);
            }
            else if(r_compare >= max_compare)
            {
                center_pos = vec2( 0.5,  0.0);
                to_px_2 = mat2(0.0, -res.x, res.y, 0.0);
            }
            else if(tl_compare >= max_compare)
            {
                center_pos = vec2(-0.5,  0.0);
                to_px_2 = mat2(-res.x,  res.x, res.y, 0.0);
            }
            else if(br_compare >= max_compare)
            {
                center_pos = vec2( 0.5,  0.0);
                to_px_2 = mat2(-res.x, -res.x, res.y, 0.0);
            }
            else if(tr_compare >= max_compare)
            {
                center_pos = vec2( 0.5,  0.0);
                to_px_2 = mat2( res.x, -res.x, res.y, 0.0);
            }
            else if(bl_compare >= max_compare)
            {
                center_pos = vec2(-0.5,  0.0);
                to_px_2 = mat2( res.x,  res.x, res.y, 0.0);
            }
            else
                skip = true;
            center_pos *= res;
            
            // follow edge and find the ratio of like to unlike pixels
            float like = 0.0;
            float unlike = 0.0;
            int used_range = 0;
            for(int y = 0; y <= 1; y += 1)
            {
                for(int x = 0; x <= int(uRange); x += 1)
                {
                    vec3 info_a = comparify(getInfo(texcoord + center_pos + vec2(float( x), float(y)-0.5)*to_px_2), center);
                    vec3 info_b = comparify(getInfo(texcoord + center_pos + vec2(float(-x), float(y)-0.5)*to_px_2), center);
                    float like_a = is_similar(info_a, mode);
                    float like_b = is_similar(info_b, mode);
                    like += like_a;
                    unlike += 1.0-like_a;
                    if(x != 0)
                    {
                        like += like_b;
                        unlike += 1.0-like_b;
                    }
                    if(like_a != like_b)
                        break;
                    if(x > used_range) used_range = x;
                }
            }
            
            // blend neighboring edge pixel into center pixel based on ratio of low to high pixels
            float blend_amount = like/(like+unlike)*2.0;
            float sharpness = mix(0.0, 1.0, clamp(float(used_range-1)/4.0, 0.0, 1.0));
            blend_amount = mix(mix(0.0, 0.5, sharpness), 1.0, clamp(blend_amount, 0.0, 1.0));
            
            vec2 dir = vec2(0.0, 0.5);
            vec4 color_a = omw_Texture2D(RT_NoMipmap, texcoord + center_pos - dir*to_px_2);
            //vec4 color_b = omw_Texture2D(RT_NoMipmap, texcoord + center_pos + dir*to_px_2);
            // color_b is always the base color in the current revision of this algorithm?
            vec4 color_b = base_color;
            if(!skip)
                omw_FragColor = mix(color_a, color_b, blend_amount);
            else
                omw_FragColor = base_color;
            
            if(uMSAACompatibilityHack)
            {
                // create an "estimate" of our blend using a neighboring pixel
                // then compare it to the actual output
                
                // luma of output
                float luma_z = dot(omw_FragColor.rgb, vec3(1.0/3.0));
                // luma of non-blend side of base color
                float luma_a = dot(omw_Texture2D(RT_NoMipmap, texcoord + center_pos + (dir*3.0)*to_px_2).rgb, vec3(1.0/3.0));
                // luma of base color
                float luma_b = dot(color_b.rgb, vec3(1.0/3.0));
                // luma of blend side of base color
                float luma_c = dot(color_a.rgb, vec3(1.0/3.0));
                
                float lite = max(max(luma_z, luma_a), max(luma_b, luma_c));
                float dark = min(min(luma_z, luma_a), min(luma_b, luma_c));
                if(lite-dark == 0.0)
                {
                    omw_FragColor = omw_FragColor;
                }
                else
                {
                    luma_z = (luma_z-dark)/(lite-dark);
                    luma_a = (luma_a-dark)/(lite-dark);
                    luma_b = (luma_b-dark)/(lite-dark);
                    luma_c = (luma_c-dark)/(lite-dark);
                    
                    // estimate of expected luma
                    float luma_q = mix(luma_a, luma_c, blend_amount);
                    
                    // compare estimate to unfiltered center pixel; if outside a certain range then replace with original color
                    float estimate_vs_base = abs(luma_q - luma_b);
                    if (estimate_vs_base > 0.5 && estimate_vs_base < 0.6)
                        omw_FragColor = base_color;
                    if (estimate_vs_base >= 0.6 && estimate_vs_base < 0.7)
                        omw_FragColor = mix(omw_FragColor, base_color, 0.5);
                    // alternatively, if the original output is closer to the estimate of the luma range than luma_z, we can assume that there was some AA already
                    if (abs(luma_b - luma_q) < abs(luma_z - luma_q))
                        omw_FragColor = base_color;
                }
            }
            
            omw_FragColor = powv(omw_FragColor, 1.0/uGamma);
            if(uOutputDebug > 0)
                omw_FragColor = vec4(vec3(0.0), 1.0);
            if(uOutputDebug == 1)
                omw_FragColor = vec4(vec3(mix(-2.0, 1.0, blend_amount)), 1.0);
            if(uOutputDebug == 2)
                omw_FragColor.rb = center_pos.xy/res+vec2(0.5);
            if(uOutputDebug == 3)
                omw_FragColor.rb = vec2(to_px_2[0][0], to_px_2[0][1])/res/2.0+vec2(0.5);
            if(uOutputDebug == 4)
                omw_FragColor.rb = vec2(to_px_2[1][0], to_px_2[1][1])/res/2.0+vec2(0.5);
            if(uOutputDebug == 5)
                omw_FragColor = powv(color_a, 1.0/uGamma);
            if(uOutputDebug == 6)
                omw_FragColor = powv(color_b, 1.0/uGamma);
            if(uOutputDebug == 7)
                omw_FragColor = vec4(vec3(used_range/uRange), 1.0);
            if(uOutputDebug == 8)
                omw_FragColor = vec4(like/(like+unlike), 0.0, 0.0, 1.0);
            if(uOutputDebug == 9)
                omw_FragColor = vec4(unlike/(like+unlike), 0.0, 0.0, 1.0);
            if(uOutputDebug == 10)
                omw_FragColor = vec4(mode, 0.0, 0.0, 1.0);
            if(uOutputDebug == 11)
                omw_FragColor = vec4(t_compare+b_compare, l_compare+b_compare, r_compare, 1.0);
            if(uOutputDebug == 12)
                omw_FragColor = vec4(t_scalar+b_scalar, l_scalar+b_scalar, r_scalar, 1.0);
            if(uOutputDebug == 13)
                omw_FragColor = vec4(tl_compare, tr_compare, bl_compare, 1.0);
            if(uOutputDebug == 14)
                omw_FragColor = vec4(omw.eyeVec.xyz*0.5+vec3(0.5), 1.0);
            
        }
        if(uOutputDebug == 15)
        {
            mat3 asdf = getInfo(texcoord);
            omw_FragColor = vec4(vec3(asdf[0][0], asdf[0][1], asdf[0][2])*0.5+vec3(0.5), 1.0);
        }
        if(uOutputDebug == 16)
            omw_FragColor = vec4(vec3(sqrt(getInfo(texcoord)[2][0])/200.0), 1.0);
    }
}

technique {
    passes = nomipmap, BAD;
    description = "Horrifying terrible no-good depth-based edge AA shader. Now with more settings. And now with normals- and colors-based edge detection (in addition to just depth).";
    author = "Wareya";
    version = "0.941";
    pass_normals = true;
}