ycbcr-to-rgb.metal

fragment float4 capturedImageFragmentShader(ImageColorInOut in [[stage_in]],
                                            texture2d<float, access::sample> capturedImageTextureY [[ texture(kTextureIndexY) ]],
                                            texture2d<float, access::sample> capturedImageTextureCbCr [[ texture(kTextureIndexCbCr) ]]) {
    
    constexpr sampler colorSampler(mip_filter::linear,
                                   mag_filter::linear,
                                   min_filter::linear);
    
    const float4x4 ycbcrToRGBTransform = float4x4(
        float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f),
        float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f),
        float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f),
        float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f)
    );
    
    // Sample Y and CbCr textures to get the YCbCr color at the given texture coordinate
    float4 ycbcr = float4(capturedImageTextureY.sample(colorSampler, in.texCoord).r,
                          capturedImageTextureCbCr.sample(colorSampler, in.texCoord).rg, 1.0);
    
    // Return converted RGB color
    return ycbcrToRGBTransform * ycbcr;
}

For UE4 ARKit plugin guys:
I wanted my object to cast shadows on detected surfaces, so one idea was to make material that takes the camera image and shows it as color on Surface Opaque material.
UE4 ARKit plugin actually does similar thing, you can see it's material if you enable Show Plugin Content, then look for AppleARKit Content / ARKitCameraMaterial.

If someone wonders how it does it:

float3 YUV_to_sRGBCustom(float4 YUV)
{
	float4x4 YCbCrConvert = 
	{ 
		1.0000, 0.0000, 1.4020, -0.7010,
		1.0000, -0.3441, -0.7141, 0.5291,
		1.0000, 1.7720, 0.0000, -0.8860,
		0.0000, 0.0000, 0.0000, 1.0000
	};

	return mul(YCbCrConvert, YUV);
}

float3 sRGB_to_linear(float3 sRGB)
{
	#if !OUTPUT_GAMMA_SPACE || METAL_MRT_PROFILE
	float3 Result = max(6.10352e-5, sRGB);
	return Result > 0.04045 ? pow( Result * (1.0 / 1.055) + 0.0521327, 2.4 ) : Result * (1.0 / 12.92);
	#else
	return sRGB;
	#endif
}