Failing benchmarkdotnet benchmarks for memstate

MemstateBenchmarks.cs

namespace Memstate.Benchmarks
{
    using System;
    using System.Linq;
    using System.Threading.Tasks;

    using BenchmarkDotNet.Attributes;
    using BenchmarkDotNet.Columns;
    using BenchmarkDotNet.Configs;

    using Memstate.EventStore;
    using Memstate.JsonNet;
    using Memstate.Models;
    using Memstate.Models.KeyValue;
    using Memstate.Postgresql;

    [Config(typeof(MemstateConfig))]
    public class MemstateBenchmarks
    {
        public const int Iterations = 10;

        private Engine<KeyValueStore<int>> _engine;

        [Params(
            //typeof(InMemoryStorageProvider),
            typeof(PostgresqlProvider),
            typeof(EventStoreProvider))]
        public Type StorageProviderTypes { get; set; }

        [GlobalSetup]
        public void Setup()
        {
            var settings = new MemstateSettings().WithRandomSuffixAppendedToStreamName();

            /* 
            var logProvider = new ConsoleLoggerProvider(filter: (cat, level) => true, includeScopes: false);
            settings.LoggerFactory.AddProvider(logProvider);
            */
            settings.StorageProvider = StorageProviderTypes.AssemblyQualifiedName;
            settings.Serializers.Register("newtonsoft.json", _ => new JsonSerializerAdapter(settings));
            var engineBuilder = new EngineBuilder(settings);
            _engine = engineBuilder.Build(new KeyValueStore<int>());
        }

        [GlobalCleanup]
        public void Cleanup()
        {
            //_engine.DisposeAsync().Wait();
        }
        
        [Benchmark(OperationsPerInvoke = Iterations)]
        public async Task CommandRoundtrip()
        {
            var tasks = Enumerable
                .Range(0, Iterations)
                .Select(i => _engine.ExecuteAsync(new Set<int>(i.ToString(), i)));

            await Task.WhenAll(tasks).ConfigureAwait(false);
        }

        private class MemstateConfig : ManualConfig
        {
            public MemstateConfig()
            {
                Add(
                    StatisticColumn.Kurtosis,
                    StatisticColumn.OperationsPerSecond,
                    StatisticColumn.P90,
                    StatisticColumn.P95);
            }
        }
    }
}

output.txt

C:\projects\memstate\Memstate.Benchmarks>dotnet run
// ***** BenchmarkRunner: Start   *****
// Found benchmarks:
//   MemstateBenchmarks.CommandRoundtrip: DefaultJob [StorageProviderTypes=Memstate.EventStore.EventStoreProvider]
//   MemstateBenchmarks.CommandRoundtrip: DefaultJob [StorageProviderTypes=Memstate.Postgresql.PostgresqlProvider]

// Validating benchmarks:
Assembly Memstate.Benchmarks which defines benchmarks references non-optimized Memstate
Assembly Memstate.Benchmarks which defines benchmarks references non-optimized Memstate.Postgresql
Assembly Memstate.Benchmarks which defines benchmarks references non-optimized Memstate.EventStore
Assembly Memstate.Benchmarks which defines benchmarks references non-optimized Memstate.JsonNet
Assembly Memstate.Benchmarks which defines benchmarks is non-optimized
// **************************
// Benchmark: MemstateBenchmarks.CommandRoundtrip: DefaultJob [StorageProviderTypes=Memstate.EventStore.EventStoreProvider]
// *** Generate ***
// Result = Success
// BinariesDirectoryPath = C:\projects\memstate\Memstate.Benchmarks\bin\Debug\netcoreapp1.1\1f88904f-3aa0-4539-a9ff-eb85910858ab\bin\Release\netcoreapp1.1

// *** Build ***
// dotnet restore took 2,79s
// dotnet build took 6,39s
// Result = Success

// *** Execute ***
// Launch: 1 / 1

// Benchmark Process Environment Information:
// Runtime=.NET Core 1.1.2 (Framework 4.6.25211.01), 64bit RyuJIT
// GC=Concurrent Workstation
// Job: DefaultJob

Pilot  1: 160 op, 160875205.79 ns, 1.0055 ms/op
Pilot  2: 320 op, 238941279.35 ns, 746.6915 us/op
Pilot  3: 640 op, 616501558.03 ns, 963.2837 us/op

IdleWarmup  1: 640 op, 13958.62 ns, 21.8104 ns/op
...
IdleWarmup  6: 640 op, 2052.74 ns, 3.2074 ns/op

IdleTarget  1: 640 op, 1231.64 ns, 1.9244 ns/op
...
IdleTarget 15: 640 op, 821.10 ns, 1.2830 ns/op

MainWarmup  1: 640 op, 403933047.87 ns, 631.1454 us/op
...
MainTarget 100: 640 op, 468181314.33 ns, 731.5333 us/op

Result  1: 640 op, 416177225.27 ns, 650.2769 us/op
...
Result 100: 640 op, 468180493.23 ns, 731.5320 us/op
GC:  588 0 0 0 64000


Mean = 750.8347 us, StdErr = 14.2391 us (1.90%); N = 100, StdDev = 142.3907 us
Min = 436.8915 us, Q1 = 646.3757 us, Median = 739.7462 us, Q3 = 865.1621 us, Max = 1,036.9283 us
IQR = 218.7864 us, LowerFence = 318.1962 us, UpperFence = 1,193.3417 us
ConfidenceInterval = [702.5425 us; 799.1270 us] (CI 99.9%), Margin = 48.2922 us (6.43% of Mean)
Skewness = 0.04, Kurtosis = 2.17


// **************************
// Benchmark: MemstateBenchmarks.CommandRoundtrip: DefaultJob [StorageProviderTypes=Memstate.Postgresql.PostgresqlProvider]
// *** Generate ***
// Result = Success
// BinariesDirectoryPath = C:\projects\memstate\Memstate.Benchmarks\bin\Debug\netcoreapp1.1\1a4f2c60-3458-4352-a038-b1b1a0208220\bin\Release\netcoreapp1.1

// *** Build ***
// dotnet restore took 3,55s
// dotnet build took 5,95s
// Result = Success

// *** Execute ***
// Launch: 1 / 1

// Benchmark Process Environment Information:
// Runtime=.NET Core 1.1.2 (Framework 4.6.25211.01), 64bit RyuJIT
// GC=Concurrent Workstation
// Job: DefaultJob

Pilot  1: 160 op, 652339096.06 ns, 4.0771 ms/op

IdleWarmup  1: 160 op, 5747.67 ns, 35.9229 ns/op
...
IdleWarmup  7: 160 op, 410.55 ns, 2.5659 ns/op

IdleTarget  1: 160 op, 821.10 ns, 5.1318 ns/op
...
IdleTarget 20: 160 op, 821.10 ns, 5.1318 ns/op

MainWarmup  1: 160 op, 625043825.98 ns, 3.9065 ms/op
...
MainWarmup  6: 160 op, 664501985.00 ns, 4.1531 ms/op

MainTarget  1: 160 op, 692820750.73 ns, 4.3301 ms/op
...
MainTarget 100: 160 op, 648408511.48 ns, 4.0526 ms/op

Result  1: 160 op, 692820217.02 ns, 4.3301 ms/op
...
Result 100: 160 op, 648407977.76 ns, 4.0525 ms/op
GC:  400 124 0 0 16000

Program.cs

namespace Memstate.Benchmarks
{
    using BenchmarkDotNet.Running;

    public class Program
    {
        public static void Main(string[] args)
        {
            var summary = BenchmarkRunner.Run<MemstateBenchmarks>();
        }
    }
}