The very simple false contour mask from Ji Won Lee, Bo Ra Lim, Rae-Hong Park, Jae-Seung Kim and Wonseok Ahn, "Two-stage false contour detection using directional contrast and its application to adaptive false contour reduction," in IEEE Transactions on Consumer Electronics, vol. 52, no. 1, pp. 179-188, Feb. 2006, doi: 10.1109/TCE.2006.1605045.

lee_banding_mask.py

from collections.abc import Callable
import vapoursynth as vs
core = vs.core
from vstools import depth, get_peak_value, get_lowest_value, get_y

def lee_banding_mask(clip: vs.VideoNode, bits: int = 2, threshold: int | None = None, edge_detection: Callable = core.std.Sobel):
    """
    False contour detection via bit depth reduction. The idea is (overly) simple: Re-quantize and find difference to input, calculate the gradient, and mark gradients under a threshold as false contours.
    Naturally, this only works on content that isn't naturally flat.

    Based on Ji Won Lee, Bo Ra Lim, Rae-Hong Park, Jae-Seung Kim and Wonseok Ahn, "Two-stage false contour detection using directional contrast and its application to adaptive false contour reduction," in IEEE Transactions on Consumer Electronics, vol. 52, no. 1, pp. 179-188, Feb. 2006, doi: 10.1109/TCE.2006.1605045.

    :param clip:           Input clip.
    :param bits:           Number of bits to reduce by.
    :param threshold:      False contour threshold.
    :param edge_detection: Edge detection function.
    """
    clip = get_y(clip)
    b = 2 ** bits
    if threshold is None:
        threshold = b
    down = depth(clip, 8, dither_type = "none")
    down = down.std.Expr(f"x {b} /")
    up = down.std.Expr(f"x {b} *", clip.format)

    diff = core.std.Expr([up, clip], "x y - abs")
    edges = edge_detection(diff)
    return core.std.Expr(edges, f"x {threshold} > {get_lowest_value(clip)} {get_peak_value(clip)} ?")