NBench Invoker Performance

ActionBenchmarkInvoker-Performance.md

BenchmarkDotNet=v0.7.8.0
OS=Microsoft Windows NT 6.2.9200.0
Processor=Intel(R) Core(TM) i7-3630QM CPU @ 2.40GHz, ProcessorCount=8
HostCLR=MS.NET 4.0.30319.42000, Arch=32-bit  [AttachedDebugger]
Type=Sdk_ActionBenchmarkInvoker  Method=InvokeRunWithContext  Mode=Throughput  .NET=HostFramework  

Platform	Jit	AvrTime	StdDev	op/s

  X64 | LegacyJit | 2.8919 ns | 0.0229 ns | 345,795,547.75 |
  X64 |    RyuJit | 2.9517 ns | 0.1514 ns | 338,792,324.73 |
  X86 | LegacyJit | 4.4625 ns | 0.0573 ns | 224,087,464.37 |

ActionBenchmarkInvoker.cs

// Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.

using System;

namespace NBench.Sdk
{
    /// <summary>
    ///     A <see cref="IBenchmarkInvoker" /> implementation that works on anonymous methods and delegates.
    /// </summary>
    public class ActionBenchmarkInvoker : IBenchmarkInvoker
    {
        /// <summary>
        ///     Default no-op action.
        /// </summary>
        public static readonly Action<BenchmarkContext> NoOp = context => { };

        private readonly Action<BenchmarkContext> _cleanupAction;
        private readonly Action<BenchmarkContext> _runAction;
        private readonly Action<BenchmarkContext> _setupAction;

        public ActionBenchmarkInvoker(string benchmarkName, Action<BenchmarkContext> runAction)
            : this(benchmarkName, NoOp, runAction, NoOp)
        {
        }

        public ActionBenchmarkInvoker(string benchmarkName,
            Action<BenchmarkContext> setupAction,
            Action<BenchmarkContext> runAction,
            Action<BenchmarkContext> cleanupAction)
        {
            BenchmarkName = benchmarkName;
            _setupAction = setupAction;
            _runAction = runAction;
            _cleanupAction = cleanupAction;
        }

        public string BenchmarkName { get; }

        public void InvokePerfSetup(BenchmarkContext context)
        {
            _setupAction(context);
        }

        public void InvokeRun(BenchmarkContext context)
        {
            _runAction(context);
        }

        public void InvokePerfCleanup(BenchmarkContext context)
        {
            _cleanupAction(context);
        }
    }
}

Benchmark_Sdk_ActionBenchmarkInvoker.cs

// Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.

using BenchmarkDotNet.Tasks;
using NBench.Sdk;

namespace NBench.Microbenchmarks.SDK
{
    [BenchmarkTask(platform: BenchmarkPlatform.X64, jitVersion: BenchmarkJitVersion.LegacyJit)]
    [BenchmarkTask(platform: BenchmarkPlatform.X86, jitVersion: BenchmarkJitVersion.LegacyJit)]
    [BenchmarkTask(platform: BenchmarkPlatform.X64, jitVersion: BenchmarkJitVersion.RyuJit)]
    public class Sdk_ActionBenchmarkInvoker
    {
        private readonly IBenchmarkInvoker _actionInvoker = new ActionBenchmarkInvoker("foo", context => {}, context => {}, context => {});

        [BenchmarkDotNet.Benchmark("How quickly can we invoke when injecting context in an ActionBenchmarkInvoker")]
        public void InvokeRunWithContext()
        {
            _actionInvoker.InvokeRun(BenchmarkContext.Empty);
        }
    }
}

Benchmark_Sdk_ReflectionBenchmarkInvoker.cs

// Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.

using System.Linq;
using BenchmarkDotNet.Tasks;
using NBench.Sdk;
using static NBench.Sdk.Compiler.ReflectionDiscovery;

namespace NBench.Microbenchmarks.SDK
{
    [BenchmarkTask(platform: BenchmarkPlatform.X64, jitVersion: BenchmarkJitVersion.LegacyJit)]
    [BenchmarkTask(platform: BenchmarkPlatform.X86, jitVersion: BenchmarkJitVersion.LegacyJit)]
    [BenchmarkTask(platform: BenchmarkPlatform.X64, jitVersion: BenchmarkJitVersion.RyuJit)]
    public class Sdk_ReflectionBenchmarkInvoker
    {
        public class BenchmarkWithContext
        {
            [PerfSetup]
            public void Setup(BenchmarkContext context)
            {
                
            }

            [PerformanceBenchmark]
            [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
            public void Run(BenchmarkContext context)
            {
                
            }

            [PerfCleanup]
            public void Cleanup(BenchmarkContext context)
            {
                
            }
        }

        public class BenchmarkWithoutContext
        {
            [PerfSetup]
            public void Setup()
            {

            }

            [PerformanceBenchmark]
            [MemoryMeasurement(MemoryMetric.TotalBytesAllocated)]
            public void Run()
            {

            }

            [PerfCleanup]
            public void Cleanup()
            {

            }
        }

        private readonly IBenchmarkInvoker _contextInvoker;
        private readonly IBenchmarkInvoker _withoutContextInvoker;

        public Sdk_ReflectionBenchmarkInvoker()
        {
            var benchmarkType = typeof (BenchmarkWithContext);
            var benchmarks = CreateBenchmarksForClass(benchmarkType);
            _contextInvoker = CreateInvokerForBenchmark(benchmarks.Single());
            _contextInvoker.InvokePerfSetup(BenchmarkContext.Empty);

            benchmarkType = typeof(BenchmarkWithoutContext);
            benchmarks = CreateBenchmarksForClass(benchmarkType);
            _withoutContextInvoker = CreateInvokerForBenchmark(benchmarks.Single());
            _withoutContextInvoker.InvokePerfSetup(BenchmarkContext.Empty);
        }

        [BenchmarkDotNet.Benchmark("How quickly can we invoke when injecting context into a ReflectionBenchmarkInvoker")]
        public void InvokeRunWithContext()
        {
            _contextInvoker.InvokeRun(BenchmarkContext.Empty);
        }

        [BenchmarkDotNet.Benchmark("How quickly can we invoke when NOT injecting context into a ReflectionBenchmarkInvoker")]
        public void InvokeRunWithoutContext()
        {
            _withoutContextInvoker.InvokeRun(BenchmarkContext.Empty);
        }
    }
}

Original-ReflectionBenchmarkInvoker-Performance.md

BenchmarkDotNet=v0.7.8.0
OS=Microsoft Windows NT 6.2.9200.0
Processor=Intel(R) Core(TM) i7-3630QM CPU @ 2.40GHz, ProcessorCount=8
HostCLR=MS.NET 4.0.30319.42000, Arch=32-bit  [AttachedDebugger]
Type=Sdk_ReflectionBenchmarkInvoker  Mode=Throughput  .NET=HostFramework  

              Method | Platform |       Jit |     AvrTime |     StdDev |         op/s |

------------------------ |--------- |---------- |------------ |----------- |------------- | InvokeRunWithContext | X64 | LegacyJit | 272.2403 ns | 15.4030 ns | 3,673,225.61 | InvokeRunWithoutContext | X64 | LegacyJit | 167.1462 ns | 4.8158 ns | 5,982,797.79 | InvokeRunWithContext | X64 | RyuJit | 262.3852 ns | 3.3146 ns | 3,811,190.68 | InvokeRunWithoutContext | X64 | RyuJit | 164.9054 ns | 4.0310 ns | 6,064,089.63 | InvokeRunWithContext | X86 | LegacyJit | 293.6151 ns | 4.0732 ns | 3,405,829.15 | InvokeRunWithoutContext | X86 | LegacyJit | 206.8702 ns | 6.2809 ns | 4,833,972.11 |

Original-ReflectionBenchmarkInvoker.cs

// Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.

using System;
using NBench.Sdk.Compiler;

namespace NBench.Sdk
{
    /// <summary>
    /// <see cref="IBenchmarkInvoker"/> implementaton that uses reflection to invoke setup / run / cleanup methods
    /// found on classes decorated with the appropriate <see cref="PerformanceBenchmarkAttribute"/>s.
    /// </summary>
    public class ReflectionBenchmarkInvoker : IBenchmarkInvoker
    {
        private readonly BenchmarkClassMetadata _metadata;
        private object _testClassInstance;

        public ReflectionBenchmarkInvoker(BenchmarkClassMetadata metadata)
        {
            _metadata = metadata;
            BenchmarkName = metadata.BenchmarkClass.FullName;
        }

        public string BenchmarkName { get; }
        public void InvokePerfSetup(BenchmarkContext context)
        {
            _testClassInstance = Activator.CreateInstance(_metadata.BenchmarkClass);
            if (_metadata.Setup.Skip) return;
            _metadata.Setup.InvocationMethod.Invoke(_testClassInstance,
                _metadata.Setup.TakesBenchmarkContext ? new object[] {context} : null);
        }

        public void InvokeRun(BenchmarkContext context)
        {
            _metadata.Run.InvocationMethod.Invoke(_testClassInstance,
                _metadata.Run.TakesBenchmarkContext ? new object[] { context } : null);
        }

        public void InvokePerfCleanup(BenchmarkContext context)
        {
            // cleanup method
            if (!_metadata.Cleanup.Skip)
            {
                _metadata.Cleanup.InvocationMethod.Invoke(_testClassInstance,
                _metadata.Cleanup.TakesBenchmarkContext ? new object[] { context } : null);
            }

            // instance cleanup
            var disposable = _testClassInstance as IDisposable;
            disposable?.Dispose();

            _testClassInstance = null;
        }
    }
}

Updated-ReflectionBenchmarkInvoker-Performance.md

BenchmarkDotNet=v0.7.8.0
OS=Microsoft Windows NT 6.2.9200.0
Processor=Intel(R) Core(TM) i7-3630QM CPU @ 2.40GHz, ProcessorCount=8
HostCLR=MS.NET 4.0.30319.42000, Arch=32-bit  [AttachedDebugger]
Type=Sdk_ReflectionBenchmarkInvoker  Mode=Throughput  .NET=HostFramework  

              Method | Platform |       Jit |   AvrTime |    StdDev |           op/s |

------------------------ |--------- |---------- |---------- |---------- |--------------- | InvokeRunWithContext | X64 | LegacyJit | 2.9036 ns | 0.0636 ns | 344,396,149.89 | InvokeRunWithoutContext | X64 | LegacyJit | 4.1686 ns | 0.0827 ns | 239,889,965.00 | InvokeRunWithContext | X64 | RyuJit | 2.9046 ns | 0.0319 ns | 344,278,130.72 | InvokeRunWithoutContext | X64 | RyuJit | 4.1809 ns | 0.1283 ns | 239,185,464.46 | InvokeRunWithContext | X86 | LegacyJit | 3.8308 ns | 0.0928 ns | 261,042,845.95 | InvokeRunWithoutContext | X86 | LegacyJit | 5.7497 ns | 0.0489 ns | 173,922,434.40 |

Updated-ReflectionBenchmarkInvoker.cs

// Copyright (c) Petabridge <https://petabridge.com/>. All rights reserved.
// Licensed under the Apache 2.0 license. See LICENSE file in the project root for full license information.

using System;
using System.Reflection;
using NBench.Sdk.Compiler;

namespace NBench.Sdk
{
    /// <summary>
    /// <see cref="IBenchmarkInvoker"/> implementaton that uses reflection to invoke setup / run / cleanup methods
    /// found on classes decorated with the appropriate <see cref="PerformanceBenchmarkAttribute"/>s.
    /// </summary>
    public class ReflectionBenchmarkInvoker : IBenchmarkInvoker
    {
        private readonly BenchmarkClassMetadata _metadata;
        private object _testClassInstance;

        private Action<BenchmarkContext> _cleanupAction;
        private Action<BenchmarkContext> _runAction;
        private Action<BenchmarkContext> _setupAction;

        public ReflectionBenchmarkInvoker(BenchmarkClassMetadata metadata)
        {
            _metadata = metadata;
            BenchmarkName = metadata.BenchmarkClass.FullName;
        }

        public string BenchmarkName { get; }
        public void InvokePerfSetup(BenchmarkContext context)
        {
            _testClassInstance = Activator.CreateInstance(_metadata.BenchmarkClass);
            _cleanupAction = Compile(_metadata.Cleanup);
            _setupAction = Compile(_metadata.Setup);
            _runAction = Compile(_metadata.Run);

            _setupAction(context);
        }

        public void InvokeRun(BenchmarkContext context)
        {
            _runAction(context);
        }

        public void InvokePerfCleanup(BenchmarkContext context)
        {
            // cleanup method
            _cleanupAction(context);

            // instance cleanup
            var disposable = _testClassInstance as IDisposable;
            disposable?.Dispose();

            _testClassInstance = null;
            _setupAction = null;
            _cleanupAction = null;
            _runAction = null;
        }

        internal static Action<BenchmarkContext> CreateDelegateWithContext(object target, MethodInfo invocationMethod)
        {
            var del =
                (Action<BenchmarkContext>)
                    Delegate.CreateDelegate(typeof (Action<BenchmarkContext>), target, invocationMethod);
            return del;
        }

        internal static Action<BenchmarkContext> CreateDelegateWithoutContext(object target, MethodInfo invocationMethod)
        {
            var del =
               (Action)
                   Delegate.CreateDelegate(typeof(Action), target, invocationMethod);

            Action<BenchmarkContext> wrappedDelegate = context => del();
            return wrappedDelegate;
        }

        private Action<BenchmarkContext> Compile(BenchmarkMethodMetadata metadata)
        {
            if (metadata.Skip)
                return ActionBenchmarkInvoker.NoOp;
            return metadata.TakesBenchmarkContext ? CreateDelegateWithContext(_testClassInstance, metadata.InvocationMethod) : CreateDelegateWithoutContext(_testClassInstance, metadata.InvocationMethod);
        }
    }
}