Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Usage: glsl-shaders="~~/SSimSuperRes.glsl"
// SSimSuperRes by Shiandow
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 3.0 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library.
//!HOOK POSTKERNEL
//!BIND HOOKED
//!SAVE LOWRES
//!HEIGHT NATIVE_CROPPED.h
//!WHEN NATIVE_CROPPED.h OUTPUT.h <
//!COMPONENTS 4
//!DESC SSSR Downscaling I
#define axis 1
#define offset vec2(0,0)
#define MN(B,C,x) (x < 1.0 ? ((2.-1.5*B-(C))*x + (-3.+2.*B+C))*x*x + (1.-(B)/3.) : (((-(B)/6.-(C))*x + (B+5.*C))*x + (-2.*B-8.*C))*x+((4./3.)*B+4.*C))
#define Kernel(x) MN(0.334, 0.333, abs(x))
#define taps 2.0
#define Luma(rgb) ( dot(rgb*rgb, vec3(0.2126, 0.7152, 0.0722)) )
vec4 hook() {
// Calculate bounds
float low = ceil((HOOKED_pos - taps/input_size) * HOOKED_size - offset - 0.5)[axis];
float high = floor((HOOKED_pos + taps/input_size) * HOOKED_size - offset - 0.5)[axis];
float W = 0.0;
vec4 avg = vec4(0);
vec2 pos = HOOKED_pos;
vec4 tex;
for (float k = low; k <= high; k++) {
pos[axis] = HOOKED_pt[axis] * (k - offset[axis] + 0.5);
float rel = (pos[axis] - HOOKED_pos[axis])*input_size[axis];
float w = Kernel(rel);
tex.rgb = textureLod(HOOKED_raw, pos, 0.0).rgb * HOOKED_mul;
tex.a = Luma(tex.rgb);
avg += w * tex;
W += w;
}
avg /= W;
return vec4(avg.rgb, abs(avg.a - Luma(avg.rgb)));
}
//!HOOK POSTKERNEL
//!BIND LOWRES
//!SAVE LOWRES
//!WIDTH NATIVE_CROPPED.w
//!HEIGHT NATIVE_CROPPED.h
//!WHEN NATIVE_CROPPED.w OUTPUT.w <
//!COMPONENTS 4
//!DESC SSSR Downscaling II
#define axis 0
#define offset vec2(0,0)
#define MN(B,C,x) (x < 1.0 ? ((2.-1.5*B-(C))*x + (-3.+2.*B+C))*x*x + (1.-(B)/3.) : (((-(B)/6.-(C))*x + (B+5.*C))*x + (-2.*B-8.*C))*x+((4./3.)*B+4.*C))
#define Kernel(x) MN(0.334, 0.333, abs(x))
#define taps 2.0
#define Luma(rgb) ( dot(rgb*rgb, vec3(0.2126, 0.7152, 0.0722)) )
vec4 hook() {
// Calculate bounds
float low = ceil((LOWRES_pos - taps/input_size) * LOWRES_size - offset - 0.5)[axis];
float high = floor((LOWRES_pos + taps/input_size) * LOWRES_size - offset - 0.5)[axis];
float W = 0.0;
vec4 avg = vec4(0);
vec2 pos = LOWRES_pos;
vec4 tex;
for (float k = low; k <= high; k++) {
pos[axis] = LOWRES_pt[axis] * (k - offset[axis] + 0.5);
float rel = (pos[axis] - LOWRES_pos[axis])*input_size[axis];
float w = Kernel(rel);
tex.rgb = textureLod(LOWRES_raw, pos, 0.0).rgb * LOWRES_mul;
tex.a = Luma(tex.rgb);
avg += w * tex;
W += w;
}
avg /= W;
return vec4(avg.rgb, abs(avg.a - Luma(avg.rgb)) + LOWRES_texOff(0).a);
}
//!HOOK POSTKERNEL
//!BIND PREKERNEL
//!SAVE varL
//!WIDTH NATIVE_CROPPED.w
//!HEIGHT NATIVE_CROPPED.h
//!WHEN NATIVE_CROPPED.h OUTPUT.h <
//!COMPONENTS 4
//!DESC SSSR varL
#define spread 1.0 / 1000.0
#define sqr(x) pow(x, 2.0)
#define GetL(x,y) PREKERNEL_tex(PREKERNEL_pt*(PREKERNEL_pos * input_size + tex_offset + vec2(x,y))).rgb
#define Gamma(x) ( pow(clamp(x, 0.0, 1.0), vec3(1.0/2.0)) )
#define Luma(rgb) ( dot(rgb*rgb, vec3(0.2126, 0.7152, 0.0722)) )
vec4 hook() {
vec3 meanL = vec3(0);
for (int X=-1; X<=1; X++)
for (int Y=-1; Y<=1; Y++) {
meanL += GetL(X,Y) * pow(spread, sqr(float(X)) + sqr(float(Y)));
}
meanL /= (1.0 + 4.0*spread + 4.0*spread*spread);
float varL = 0.0;
for (int X=-1; X<=1; X++)
for (int Y=-1; Y<=1; Y++) {
varL += Luma(abs(GetL(X,Y) - meanL)) * pow(spread, sqr(float(X)) + sqr(float(Y)));
}
varL /= (spread + 4.0*spread + 4.0*spread*spread);
return vec4(GetL(0,0), varL);
}
//!HOOK POSTKERNEL
//!BIND LOWRES
//!SAVE varH
//!WIDTH NATIVE_CROPPED.w
//!HEIGHT NATIVE_CROPPED.h
//!WHEN NATIVE_CROPPED.h OUTPUT.h <
//!COMPONENTS 1
//!DESC SSSR varH
#define spread 1.0 / 1000.0
#define sqr(x) pow(x, 2.0)
#define GetH(x,y) LOWRES_texOff(vec2(x,y)).rgb
#define Gamma(x) ( pow(clamp(x, 0.0, 1.0), vec3(1.0/2.0)) )
#define Luma(rgb) ( dot(rgb*rgb, vec3(0.2126, 0.7152, 0.0722)) )
vec4 hook() {
vec3 meanH = vec3(0);
for (int X=-1; X<=1; X++)
for (int Y=-1; Y<=1; Y++) {
meanH += GetH(X,Y) * pow(spread, sqr(float(X)) + sqr(float(Y)));
}
meanH /= (1.0 + 4.0*spread + 4.0*spread*spread);
float varH = 0.0;
for (int X=-1; X<=1; X++)
for (int Y=-1; Y<=1; Y++) {
varH += Luma(abs(GetH(X,Y) - meanH)) * pow(spread, sqr(float(X)) + sqr(float(Y)));
}
varH /= (spread + 4.0*spread + 4.0*spread*spread);
return vec4(varH, 0, 0, 0);
}
//!HOOK POSTKERNEL
//!BIND HOOKED
//!BIND LOWRES
//!BIND varL
//!BIND varH
//!WHEN NATIVE_CROPPED.h OUTPUT.h <
//!DESC SSSR final pass
// -- Window Size --
#define taps 3.0
#define even (taps - 2.0 * floor(taps / 2.0) == 0.0)
#define minX int(1.0-ceil(taps/2.0))
#define maxX int(floor(taps/2.0))
#define factor (LOWRES_pt*HOOKED_size)
#define Kernel(x) (cos(acos(-1.0)*(x)/taps)) // Hann kernel
#define sqr(x) dot(x,x)
// -- Input processing --
#define L(x,y) ( varL_tex(varL_pt*(pos+vec2(x,y)+0.5)) )
#define H(x,y) ( varH_tex(varH_pt*(pos+vec2(x,y)+0.5)) )
#define Lowres(x,y) ( LOWRES_tex(LOWRES_pt*(pos+vec2(x,y)+0.5)) )
#define Gamma(x) ( pow(clamp(x, 0.0, 1.0), vec3(1.0/2.0)) )
#define GammaInv(x) ( pow(clamp(x, 0.0, 1.0), vec3(2.0)) )
#define Luma(rgb) ( dot(rgb*rgb, vec3(0.2126, 0.7152, 0.0722)) )
vec4 hook() {
vec4 c0 = HOOKED_tex(HOOKED_pos);
// Calculate position
vec2 pos = HOOKED_pos * LOWRES_size - vec2(0.5);
vec2 offset = pos - (even ? floor(pos) : round(pos));
pos -= offset;
vec2 mVar = vec2(0.0);
for (int X=-1; X<=1; X++)
for (int Y=-1; Y<=1; Y++) {
vec2 w = clamp(1.5 - abs(vec2(X,Y) - offset), 0.0, 1.0);
mVar += w.r * w.g * vec2(Lowres(X,Y).a, 1.0);
}
mVar.r /= mVar.g;
// Calculate faithfulness force
float weightSum = 0.0;
vec3 diff = vec3(0);
for (int X = minX; X <= maxX; X++)
for (int Y = minX; Y <= maxX; Y++)
{
float varL = L(X,Y).a;
float varH = H(X,Y).r;
float R = -sqrt((varL + sqr(0.5/255.0)) / (varH + mVar.r + sqr(0.5/255.0)));
vec2 krnl = Kernel(vec2(X,Y) - offset);
float weight = krnl.r * krnl.g / (Luma(abs(c0.rgb - Lowres(X,Y).rgb)) + Lowres(X,Y).a + sqr(0.5/255.0));
diff += weight * (L(X,Y).rgb + Lowres(X,Y).rgb * R + (-1.0 - R) * (c0.rgb));
weightSum += weight;
}
diff /= weightSum;
c0.rgb = ((c0.rgb) + diff);
return c0;
}
@haasn

This comment has been minimized.

Copy link

@haasn haasn commented May 15, 2016

//!SAVE discard is no longer necessary. mpv automatically saves textures if something else depends on it.

@haasn

This comment has been minimized.

Copy link

@haasn haasn commented May 17, 2016

LINEAR and SCALED are not good combinations. They may be in different colorspaces (SCALED can be in gamma light rather than linear light, and will also include subtitles). (Also, there's no guarantee LINEAR will ever fire)

Try PREKERNEL and POSTKERNEL, they are probably ideal for your use case.

@haasn

This comment has been minimized.

Copy link

@haasn haasn commented May 18, 2016

FWIW, this and similar scalers also completely breaks when the POSTKERNEL and PREKERNEL textures don't exactly line up (e.g. cropping with panscan, or when using a prescaler that has a nonzero offset alongside it).

I'm still not entirely sure what to do about this. Either we force POSTKERNEL to always render the full size, but that has some really bad consequences; or we force PREKERNEL to get loaded through the same transformation (but that also has some really bad consequences).

@Becauseitsbaby

This comment has been minimized.

Copy link

@Becauseitsbaby Becauseitsbaby commented Dec 18, 2016

Hello, I stopped by to say that I appreciate your work, but also have a question. Please tell me if this is in the wrong place btw. If I wanted to upscale 720p to 1080p, can I change the setting of the script so that I could do it without downsampling it back?

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Dec 20, 2016

Nope, you can't, the only settings you can change are localities and oversharp.

@Becauseitsbaby

This comment has been minimized.

Copy link

@Becauseitsbaby Becauseitsbaby commented Dec 22, 2016

Gotcha, thanks for the response.

@haasn

This comment has been minimized.

Copy link

@haasn haasn commented Aug 3, 2017

I'm still not entirely sure what to do about this. Either we force POSTKERNEL to always render the full size, but that has some really bad consequences; or we force PREKERNEL to get loaded through the same transformation (but that also has some really bad consequences).

Incididentally, I was thinking about how to solve this and thought that maybe we should “normalize” every texture to the same coordinate space as HOOKED. So if you hook POSTKERNEL and bind PREKERNEL, the same crop/transformation would be applied to the prekernel bind. Similarly, if you hook LUMA and bind CHROMA, you would get the chroma texture aligned to the LUMA texture. Would this be a better API for you?

Also, not sure if your shader does this currently or not, but for your/future reference you can calculate the correct prekernel position like this:

vec2 kpos = (input_size * PREKERNEL_pos + tex_offset) / PREKERNEL_size;

This works even when using panning/cropping, and allows you to directly compare POSTKERNEL against PREKERNEL.

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Aug 4, 2017

vec2 kpos = (input_size * PREKERNEL_pos + tex_offset) * PREKERNEL_pt

Yes, I use this since March.

Similarly, if you hook LUMA and bind CHROMA, you would get the chroma texture aligned to the LUMA texture.

This would be useful I guess. What about binding luma prescaled with nnedi3 for example to original LUMA?

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Jan 21, 2018

This shader only works when mpv buil-in scaler is used and the only other way to make it faster is to use a less demanding scaling algorithm, like spline36 instead of ewa_lanczos, or bicubic_fast instead of spline36.

Does this shader run slower if the video has a high framerate/resolution?

Yes, of course.

Can I use that filter along with this shader to get a better image?

If your scaling factor is 2x or smaller this shader won't be applied, only nnedi3.

Can other vapoursynth filters(like psharpen) be used along with this shader?

Yes.

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Jan 22, 2018

Is this understanding correct?

Yes

Finally, does a lot of motion in the video or lots of dark areas in the video have any effect on how fast this shader runs.

No, it shouldn't.

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Jun 18, 2018

This is most likely the final revision, I'm finally happy with its quality and performance (1 pass is enough now).

@zc62

This comment has been minimized.

Copy link

@zc62 zc62 commented Jan 3, 2019

@igv It seems that your adaptive-antiringing.glsl is deleted? Is there any reason for that? (I couldn't find a proper place to report this, so I am leaving a comment here.)

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Jan 3, 2019

It was experimental and didn't work well enough, and I didn't have much desire to work on it, so I removed it, use SSSR instead.

@deus0ww

This comment has been minimized.

Copy link

@deus0ww deus0ww commented Apr 25, 2019

"To make it sharper use Mitchell params instead of Catmull-Rom..." Is that in reference to #define Kernel(x)... in line 31 and 79?

Would Robidoux params be even sharper (than Mitchell)?

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Apr 26, 2019

Yes.
Yes, Robidoux is slightly sharper than Mitchell.

@Tsubajashi

This comment has been minimized.

Copy link

@Tsubajashi Tsubajashi commented Apr 29, 2019

which params would be for "Robidoux"?
i enjoy a tiny bit sharper videos than default SSSR.

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Apr 29, 2019

0.3782, 0.3109

@deus0ww

This comment has been minimized.

Copy link

@deus0ww deus0ww commented Apr 30, 2019

Params:

RobidouxSoft
B = (9-3*sqrt(2))/7 = 0.67962275898295921
C = 0.1601886205085204

Robidoux
B = 12/(19+9sqrt(2)) = 0.37821575509399866
C = 113/(58+216
sqrt(2)) = 0.31089212245300067

Mitchell
B = 1/3
C = 1/3

RobidouxSharp
B = 6/(13+7sqrt(2)) = 0.2620145123990142
C = 7/(2+12
sqrt(2)) = 0.3689927438004929

Catrom
B = 0.0
C = 0.5

Sources:
https://www.imagemagick.org/Usage/filter/#mitchell
https://www.imagemagick.org/discourse-server/viewtopic.php?f=22&t=19823

@dixie-flatliner

This comment has been minimized.

Copy link

@dixie-flatliner dixie-flatliner commented Apr 22, 2020

Does SSIMSuperRes require use of an upscaling filter which matches the one in the shader, as SSIMDownscaler does?

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Apr 23, 2020

No.

@po5

This comment has been minimized.

Copy link

@po5 po5 commented Jul 3, 2020

This stretches and crops videos with a display aspect ratio when used together with FSRCNNX.

glsl-shaders="~~/shaders/FSRCNNX_x2_8-0-4-1.glsl"
glsl-shaders-append="~~/shaders/SSimSuperRes.glsl"

Sample video: https://0x0.st/iyNb.mkv generated with the following commands
wget http://distribution.bbb3d.renderfarming.net/video/mp4/bbb_sunflower_1080p_60fps_normal.mp4
ffmpeg -ss 1:10 -t 5 -r 25 -i bbb_sunflower_1080p_60fps_normal.mp4 -vf scale=720:576 bbb.mkv

MediaInfo:

General
Unique ID                                : 174675382607623785347252629000047310909 (0x83694147194CB82603DFF87EA3F4383D)
Complete name                            : bbb.mkv
Format                                   : Matroska
Format version                           : Version 4
File size                                : 404 KiB
Duration                                 : 5 s 3 ms
Overall bit rate mode                    : Variable
Overall bit rate                         : 662 kb/s
Movie name                               : Big Buck Bunny, Sunflower version
Writing application                      : Lavf58.29.100
Writing library                          : Lavf58.29.100
Comment                                  : Creative Commons Attribution 3.0 - http://bbb3d.renderfarming.net
ErrorDetectionType                       : Per level 1
ARTIST                                   : Blender Foundation 2008, Janus Bager Kristensen 2013
COMPOSER                                 : Sacha Goedegebure
GENRE                                    : Animation

Video
ID                                       : 1
Format                                   : AVC
Format/Info                              : Advanced Video Codec
Format profile                           : High@L3
Format settings                          : CABAC / 4 Ref Frames
Format settings, CABAC                   : Yes
Format settings, Reference frames        : 4 frames
Codec ID                                 : V_MPEG4/ISO/AVC
Duration                                 : 5 s 3 ms
Bit rate                                 : 373 kb/s
Width                                    : 720 pixels
Height                                   : 576 pixels
Display aspect ratio                     : 16:9
Frame rate mode                          : Constant
Frame rate                               : 25.000 FPS
Standard                                 : PAL
Color space                              : YUV
Chroma subsampling                       : 4:2:0
Bit depth                                : 8 bits
Scan type                                : Progressive
Bits/(Pixel*Frame)                       : 0.036
Stream size                              : 228 KiB (56%)
Writing library                          : x264 core 159 r2999 296494a
Encoding settings                        : cabac=1 / ref=3 / deblock=1:0:0 / analyse=0x3:0x113 / me=hex / subme=7 / psy=1 / psy_rd=1.00:0.00 / mixed_ref=1 / me_range=16 / chroma_me=1 / trellis=1 / 8x8dct=1 / cqm=0 / deadzone=21,11 / fast_pskip=1 / chroma_qp_offset=-2 / threads=12 / lookahead_threads=2 / sliced_threads=0 / nr=0 / decimate=1 / interlaced=0 / bluray_compat=0 / constrained_intra=0 / bframes=3 / b_pyramid=2 / b_adapt=1 / b_bias=0 / direct=1 / weightb=1 / open_gop=0 / weightp=2 / keyint=250 / keyint_min=25 / scenecut=40 / intra_refresh=0 / rc_lookahead=40 / rc=crf / mbtree=1 / crf=23.0 / qcomp=0.60 / qpmin=0 / qpmax=69 / qpstep=4 / ip_ratio=1.40 / aq=1:1.00
Default                                  : Yes
Forced                                   : No

Audio
ID                                       : 2
Format                                   : Vorbis
Format settings, Floor                   : 1 / 16898
Codec ID                                 : A_VORBIS
Duration                                 : 5 s 3 ms
Bit rate mode                            : Variable
Bit rate                                 : 276 kb/s
Channel(s)                               : 6 channels
Sampling rate                            : 48.0 kHz
Compression mode                         : Lossy
Stream size                              : 169 KiB (42%)
Writing application                      : Lavc58.54.100
Writing library                          : libVorbis (Now 100% fewer shells) (20180316 (Now 100% fewer shells))
Default                                  : Yes
Forced                                   : No

Screenshots (in fullscreen, to trigger scaling):
FSRCNNX
FSRCNNX+SSimSuperRes

As you can see it's stretched vertically and part of the image is missing at the bottom.

This was caused by commit 44add43ba89236b107431c375e3c98a8841f0065. The previous commit (5431ce139c20a9dccd1d3eaa0aee1cb2a2647771) works fine.

This display aspect ratio stuff is common in PAL regions, please fix.

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Jul 3, 2020

Fixed.

@crazysword1

This comment has been minimized.

Copy link

@crazysword1 crazysword1 commented Feb 6, 2021

Hi igv,

I am currently using all your shaders. My current config is:

scale=lanczos
dscale=mitchell
cscale=lanczos

For scale and cscale , does it matter what I use at all? I am already using FSRCNNX + SSSR and for chroma, I am already using Krig.
I know dscale should be Mitchell because I am using the ssimdownscaler.

Thanks

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Feb 6, 2021

For small upscaling ratios (when FSRCNNX doesn't kick in) scale=ewa_hanning or similar might be sometimes slightly better than lanczos.
cscale doesn't matter.

@crazysword1

This comment has been minimized.

Copy link

@crazysword1 crazysword1 commented Feb 6, 2021

Thanks for the quick response. So SSimSuperRes does not replace scale, but it's a supplement/tweak to it? I always thought SSimSuperRes would kick in for small upscaling ratios ( <1.4 for FSRCNNX ) and it was to replace the MPV's built-in scale. Is SSimSuperRes also a 2x prescaler like FSRCNNX?

@igv

This comment has been minimized.

Copy link
Owner Author

@igv igv commented Feb 7, 2021

It's a supplement/tweak to scale.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment