MemoryManager.cs

public class Configuration
{    
    public MemoryManager MemoryManager { get; set; } = SharedPoolingManagedMemoryManager.Instance;

    public static Configuration Default { get; }

    // ...
}

public interface IDecoderOptions
{
    // Allows setting a custom MemoryManager for an image decoding operation.
    // When MemoryManagerOverride == null, algorithms should use the MemoryManager specified in Configuration 
    MemoryManager MemoryManagerOverride { get; set; } = null;
    
    // ...
}

public abstract class MemoryManager
{
    // "Pool" is an abstract term here. The concrete meaning might be implementation specific.
    // Eg. for PoolingManagedMemoryManager it means the ArrayPool-s.
    // In a future MemoryManager implementation it might mean unmanaged memory pools.
    public virtual void Reset() { }

    // All implementation deatails are hidden for now.
    // To make MemoryManager extensible by users, we will need the new standard corefx (and maybe even corefxlab!) libraries.
    // corefx Sytem.Memory will be released with .NET Core 2.0
}

// Allocates managed arrays without pooling
public class BasicManagedMemoryManager
{
}

// Allocates managed arrays, using ArrayPool-s
public abstract class PoolingManagedMemoryManager
{
    protected abstract ArrayPool<T> GetArrayPool<T>();
}

public sealed class SharedPoolingManagedMemoryManager : PoolingManagedMemoryManager
{
    public static SharedPoolingManagedMemoryManager Instance { get; } = new SharedPoolingManagedMemoryManager();

    public static int MaximumPooledArraySizeInBytes { get; set; } // Setter should throw when calling after the first actual usage of SharedPoolingManagedMemoryManager

    // Use the only true singleton instance plz!
    private SharedPoolingManagedMemoryManager() { }

    protected override ArrayPool<T> GetArrayPool<T>() => ArrayPoolSingletons<T>.ArrayPool;

    public override Reset()
    {
        // Implementation is gonna be tricky here, but possible
    }

    class ArrayPoolSingletons<T>
        where T : struct
    {
        public static ArrayPool<T> ArrayPool { get; } = // Create default instance
    }
}


// Allocates "memory" bound to Memory Mapped Files
public class MMFMemoryManager
{
}

// Allows building custom MemoryManagers with a fluent API 
public class CombinedMemoryManager
{
    public CombinedMemoryManager(MemoryManager defaultMemoryManager) { }

    public CombinedMemoryManager OverrideBelow(int byteLimit, MemoryManager mmOverride) { }

    public CombinedMemoryManager OverrideOver(int byteLimit, MemoryManager mmOverride) { }
}


public class Examples
{
    public void ResetDefaultArrayPools()
    {
        // The default MemoryManager is OK for me, but the pools are grown too big, I want to flush their contents!
        
        SharedPoolingManagedMemoryManager.Instance.Reset();
        GC.Collect();
    }

    public void NoArrayPools()
    {
        // A simple solution for ArrayPool haters:

        Configuration.Default.MemoryManager = new BasicManagedMemoryManager();
    }

    public void UseCustomCustomConfiguration()
    {
        // My application is big, the code needs to be robust! 
        // I don't want to depend on singletons like Configuration.Default.
        // And yeah, I also want to set my custom MemoryManager!

        class MyApplication
        {
            public Configuration ImageSharpConfiguration { get; } = // Build your own config here!
            public Image LoadImage(string path) => Image.Load(this.Configuration, path);
        }
    }

    public void TurnOffArrayPoolsForJpeg()
    {
        // I'm fine with ArrayPool-s, but random private jpeg data structures are camping in my memory!
        // I want to disable pooling for them! 

        JpegDecoderOptions options = new JpegDecoderOptions()
        {
            MemoryManagerOverride = new BasicManagedMemoryManager()
        };

        Image.Load("foo.jpg", options);
    }

    public void CustomArrayPoolLimit()
    {
        // I don't want arrays over 4K to be pooled!

        // 1. Configuring the default shared memory manager:
        SharedPoolingManagedMemoryManager.MaximumPooledArraySizeInBytes = 4096;

        // 2. Creating a CombinedMemoryManager:
        CombinedMemoryManager customMgr = 
            new CombinedMemoryManager(SharedPoolingManagedMemoryManager.Instance)
            .OverrideOver(4096, new BasicManagedMemoryManager());

        Configuration.Default.MemoryManager = customMgr;
    }

    public void CustomArrayPool()
    {
        // I want to use my own ArrayPool implementation

        class MyPoolingMemoryManager : PoolingManagedMemoryManager
        {
            protected override ArrayPool<T> GetArrayPool<T>()
            {
                // Implement your own array pool creation and caching logic
            }
        }

        Configuration.Default.MemoryManager = new MyPoolingMemoryManager();
    }

    public void JpegMMF()
    {
        // Private stuff in jpeg decoder is allocating too much.
        // I want to utilize Memory Mapped Files to reduce it, with the cost of lower decoding speed!

        class MyApplication
        {
            public MemoryManager JpegMemoryManager { get; } = 
                new CombinedMemoryManager(Configuration.Default.MemoryManager)
                .OverrideOver(1000*1000*sizeof(Color)*4, new MMFMemoryManager());

            public Image LoadJpeg(string path) => Image.Load(path, new JpegDecoderOptions() { MemoryManagerOverride = JpegMemoryManager});
        }
    }

    public void UseMMFWithImages()
    {
        // Q: This MMF stuff is cool! I don't have much memory, so I want the Image class to also use MMF!

        // A: This would be possible in the future:
        Configuration.Default.MemoryManager = new MMFMemoryManager();

        // But not now:
        Image img = new Image(1000, 1000); // BANG! This will throw a NotSupportedException!

        // We need to wait for corefx / corefxlab libraries to enable this feature.
    }
}