NUnit.Bench.Ratio.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;
using NUnit.Framework;

namespace CodeDrafts.PerfTests
{
	public class SingleRunFastConfig : ManualConfig
	{
		public SingleRunFastConfig()
		{
			Add(Job.Dry);
		}
	}

	[Config(typeof(SingleRunFastConfig))]
	public class PerfAssertions
	{
		#region Infrastructure
		// BASEDON: http://stackoverflow.com/a/8137526
		private static double Percentile(IEnumerable<double> sequence, double percentileFraction)
		{
			var temp = sequence.ToArray();
			Array.Sort(temp);

			int N = temp.Length;
			double n = (N - 1) * percentileFraction + 1;
			if (n == 1d) return temp[0];
			else if (n == N) return temp[N - 1];
			else
			{
				int k = (int)n;
				double d = n - k;
				return temp[k - 1] + d * (temp[k] - temp[k - 1]);
			}
		}

		private static double L95Percentile(BenchmarkReport report)
		{
			var ns = report.AllMeasurements.Select(m => m.Nanoseconds);
			return Percentile(ns, 0.95);
		}

		// BASEDON: https://github.com/PerfDotNet/BenchmarkDotNet/blob/master/BenchmarkDotNet.IntegrationTests/PerformanceUnitTest.cs
		// TODO: TestSets: compile multiple bencmark types at once to minigate compilation overhead?
		/// <summary>
		/// Trying to make it work
		/// </summary>
		public void TestPerf<T>(double maxRatio, string messageFormat)
		{
			var curDir = Environment.CurrentDirectory;
			try
			{
				// WORKAROUND: fixing https://github.com/nunit/nunit3-vs-adapter/issues/96
				Environment.CurrentDirectory = TestContext.CurrentContext.TestDirectory;

				var summary = BenchmarkRunner.Run<T>();

				var baselineBenchmark = summary.Benchmarks.FirstOrDefault(b => b.Target.Baseline);
				if (baselineBenchmark == null)
				{
					Assert.True(baselineBenchmark != null, "Baseline not defined");
				}
				var baselineMetric = L95Percentile(summary.Reports[baselineBenchmark]);
				var reports = summary.Reports.Values;
				foreach (var report in reports)
				{
					if (report.Benchmark == baselineBenchmark) continue;

					var reportMetric = L95Percentile(report);
					var ratio = Math.Round(reportMetric / baselineMetric, 2);

					Assert.That(ratio < maxRatio, messageFormat, report.Benchmark.ShortInfo, maxRatio, ratio);
				}
			}
			finally
			{
				Environment.CurrentDirectory = curDir;
			}
		}
		#endregion

		[Test]
		public void TestNotSlowerThan10()
		{
			TestPerf<PerfAssertions>(2.0, "Bench {0} is slower than {1}x baseline. Actual: {2}x");
		}

		[Benchmark(Baseline = true)]
		public void FastBenchmark()
		{
			Thread.Sleep(10);
		}
		[Benchmark]
		public void FasterBenchmark()
		{
			// Do nothing;
		}
		[Benchmark]
		public void SlowBenchmark()
		{
			Thread.Sleep(90);
		}
	}
}