Frei-Chen edge detection in luma (mpv .hook shader)

edge_freichen.hook

//!HOOK LUMA
//!BIND HOOKED
//!DESC Frei-Chen Edge Detection

/*
Frei-Chen Edge Detection (in Luma)
original source code (freichen.frag): http://rastergrid.com/blog/2011/01/frei-chen-edge-detector/
glsl-shader adapted to mpv .hook by butterw

best results as a LUMA hook but can also work on MAIN or OUTPUT, can be combined with NoChroma.hook.
*/

const mat3 G[9] = mat3[](0.5*sqrt(2.) * mat3( 1.0, sqrt(2.), 1., 0., 0., 0., -1., -sqrt(2.), -1. ), 0.5*sqrt(2.) * mat3( 1.0, 0.0, -1.0, sqrt(2.), 0.0, -sqrt(2.), 1.0, 0.0, -1. ),	0.5*sqrt(2.) * mat3( 0.0, -1.0, sqrt(2.), 1.0, 0.0, -1.0, -sqrt(2.), 1.0, 0. ),	0.5*sqrt(2.) * mat3( sqrt(2.), -1.0, 0.0, -1.0, 0.0, 1.0, 0.0, 1.0, -sqrt(2.) ), 0.5*mat3( 0.0, 1.0, 0.0, -1.0, 0.0, -1.0, 0.0, 1.0, 0.0 ),	0.5*mat3( -1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, -1.0 ), 1/6.*mat3( 1.0, -2.0, 1.0, -2.0, 4.0, -2.0, 1.0, -2.0, 1.0 ), 1/6.*mat3( -2.0, 1.0, -2.0, 1.0, 4.0, 1.0, -2.0, 1.0, -2.0 ), 1/3.*mat3( 1., 1., 1., 1., 1., 1., 1., 1., 1. ));

vec4 hook(){
	mat3 I;
	float cnv[9];
	vec3 sample0;
	
	/* fetch the 3x3 neighbourhood and use the RGB vector's length as intensity value */
	for (int i=0; i<3; i++){
		for (int j=0; j<3; j++) {
		sample0 = HOOKED_texOff(vec2(i-1, j-1)).rgb;
		I[i][j] = length(sample0); 
		}
	}
	
	/* calculate the convolution values for all the masks */
	for (int i=0; i<9; i++) {
		float dp3 = dot(G[i][0], I[0]) + dot(G[i][1], I[1]) + dot(G[i][2], I[2]);
		cnv[i] = dp3 * dp3; 
	}

	float M = (cnv[0] + cnv[1]) + (cnv[2] + cnv[3]);
	float S = (cnv[4] + cnv[5]) + (cnv[6] + cnv[7]) + (cnv[8] + M); 
	
	return vec4(sqrt(M/S));
}