Code that fails in Release only.

JitFail.cs

        private void ProcessScanlineFromPalette<TPixel>(ref byte scanlineRef, ref TPixel rowRef)
            where TPixel : struct, IPixel<TPixel>
        {
            ReadOnlySpan<Rgb24> palettePixels = MemoryMarshal.Cast<byte, Rgb24>(this.palette);
            ref Rgb24 palettePixelsRef = ref MemoryMarshal.GetReference(palettePixels);

            if (this.paletteAlpha?.Length > 0)
            {
                // If the alpha palette is not null and has one or more entries, this means, that the image contains an alpha
                // channel and we should try to read it.
                var pixel = default(TPixel);
                Rgba32 rgba = default;
                ref byte paletteAlphaRef = ref this.paletteAlpha[0];

                for (int x = 0; x < this.header.Width; x++)
                {
                    int index = Unsafe.Add(ref scanlineRef, x);
                    rgba.Rgb = Unsafe.Add(ref palettePixelsRef, index);
                    rgba.A = this.paletteAlpha.Length > index ? Unsafe.Add(ref paletteAlphaRef, index) : byte.MaxValue;

                    pixel.PackFromRgba32(rgba);
                    Unsafe.Add(ref rowRef, x) = pixel; 
                }
            }
            else
            {
                // TODO: We should have PackFromRgb24.
                var pixel = default(TPixel);
                var rgba = new Rgba32(0, 0, 0, byte.MaxValue);
                for (int x = 0; x < this.header.Width; x++)
                {
                    int index = Unsafe.Add(ref scanlineRef, x);
                    rgba.Rgb = Unsafe.Add(ref palettePixelsRef, index);

                    pixel.PackFromRgba32(rgba);
                    Unsafe.Add(ref rowRef, x) = pixel;
                }
            }
        }

If we pass a Span<TPixel> to the methods and use row[x] = pixel instead of Unsafe.Add(ref rowRef, x) = pixel; the code will pass in both Debug and Release.