Quick n Dirty PHP Benchmark Runner

README.md

Quick n' Dirty PHP Benchmark Runner

This is a quick code profiler/benchmarker for PHP functions. Instantiate the class, load it with the tests you require, then let it run.

Example

<?php
$runner = new Runner();

// NOTE: Test parameters are passed with an array, as per call_user_func_array.
$runner->setTestParameters([eval(str_replace('#', 'null', 'return [[#,#,#],[#' .
'],[],#,[["name"=>"PHP_VERSION","value"=>"8.3.7"],["name"=>"PHP_MAJOR_VERSION' .
'","value"=>8],["name"=>"PHP_MINOR_VERSION","value"=>3],["name"=>"PHP_RELEASE' .
'_VERSION","value"=>7],["name"=>"PHP_EXTRA_VERSION","value"=>""],["name"=>"PH' .
'P_VERSION_ID","value"=>80307],["name"=>"PHP_ZTS","value"=>0]],["name"=>"PHP_' .
'SAPI","value"=>""],[#],["name"=>"PHP_BINARY","value"=>""],[],#,[],[],[],[#,#' .
',#,#,#,#,#,#],[],[#],[#,#,#,#,#,#,#],[],[#,#],#,#,#,#,[#],#,#];'
))]);

$runner->addTestCase('foreach-loop', function ($constants) {
    foreach ($constants as $index => &$constant) {
        if (is_null($constant)) {
            unset($constants[$index]);
            continue;
        }

        // If this is a constant registration value, skip it.
        if (array_key_exists('name', $constant)) {
            continue;
        }

        // Removes NULL values from the second level of the array.
        $constants[$index] = array_filter($constants[$index]);

        // If we now have an empty array, we should remove it altogether.
        if (empty($constants[$index])) {
            unset($constants[$index]);
            continue;
        }
    }
});
$runner->addTestCase('functional', function ($constants) {
    // Removes NULL values from the first level of the array.
    $constants = array_filter($constants);

    // Removes all NULL values from the second level of the array.
    $constants = array_map('array_filter', $constants);

    // Removes all empty second level arrays.
    $constants = array_filter($constants);
});
$runner->addTestCase('example-really-quick', function ($constants) {
    // Do nothing
});
$runner->run();

php test.php 
Beginning tests. Tests may run up to 10 seconds.

 +----------------+--------------+------------+----------------------+
 |                | foreach-loop | functional | example-really-quick |
 +----------------+--------------+------------+----------------------+
 |       numTests |          389 |        389 |                  389 | 
 |  averageMemory |        1.2KB |     3.57KB |                 256B | 
 |    totalMemory |     468.23KB |     1.36MB |              97.25KB | 
 | averageRuntime |      21.55ms |     4.29ms |             941.26us | 
 |   totalRuntime |        8.38s |      1.67s |             366.15ms | 
 +----------------+--------------+------------+----------------------+


Total test time: 10.42s

Runner.php

<?php
/**
 * This is a quick and dirty code benchmarker for PHP functions. It is not
 * intended to replace a proper benchmark suite.
 *
 * @author Shane Thompson
 */
declare(strict_types=1);

namespace shanept\Benchmark;

class Runner
{
    protected $testData = null;

    protected $testCase = [];

    protected $runtimeStatistics = [];

    private $mulFn = null;
    private $subFn = null;

    public function setTestParameters($parameterList)
    {
        $this->testData = $parameterList;
    }

    public function addTestCase(string $name, callable $testCase)
    {
        $this->testCase[$name] = $testCase;
    }

    public function output($timeToRun)
    {
        $results = [
            'numTests' => [],
            'averageMemory' => [],
            'totalMemory' => [],
            'averageRuntime' => [],
            'totalRuntime' => [],
        ];

        $valueMaps = [
            'numTests' => 'intval',
            'averageMemory' => [__CLASS__, 'bytes2hr'],
            'totalMemory' => [__CLASS__, 'bytes2hr'],
            'averageRuntime' => [__CLASS__, 'nanosecs2hr'],
            'totalRuntime' => [__CLASS__, 'nanosecs2hr'],
        ];

        $columns = array_keys($this->testCase);
        $numColumns = count($columns);
        $columnWidths = array_map('strlen', $columns);
        $longestMetricName = max(array_map('strlen', array_keys($results)));

        foreach ($this->runtimeStatistics as $name => $statistic) {
            $result = $this->evaluateStatsForTestCase($name, $statistic);

            $results['numTests'][$name] = $result['numTests'];
            $results['averageMemory'][$name] = $result['averageMemory'];
            $results['totalMemory'][$name] = $result['totalMemory'];
            $results['averageRuntime'][$name] = $result['averageRuntime'];
            $results['totalRuntime'][$name] = $result['totalRuntime'];
        }

        echo "\n";
        $divider = ' +' . str_repeat('-', $longestMetricName + 2) . '+';
        for ($i = 0; $i < $numColumns; $i++) {
            $divider .= str_repeat('-', $columnWidths[$i] + 2);
            $divider .= '+';
        }
        $divider .= "\n";

        // Output headers.
        echo $divider;
        echo ' |' . str_repeat(' ', $longestMetricName + 2);
        printf("| %s |\n", join(' | ', $columns));
        echo $divider;

        // Output Data
        foreach ($results as $testType => $result) {
            printf(' | %' . $longestMetricName . 's | ', $testType);

            for ($i = 0; $i < $numColumns; $i++) {
                $outputValue = call_user_func($valueMaps[$testType], $result[$columns[$i]]);

                $fmt = sprintf('%%%ds | ', $columnWidths[$i]);

                printf($fmt, $outputValue);
            }

            echo "\n";
        }

        echo $divider;
        echo "\n\nTotal test time: " . round($timeToRun, 2) . "s\n";
    }

    protected function evaluateStatsForTestCase($name, $stats)
    {
        $numTests = count($this->runtimeStatistics[$name]);
        $totalMemory = 0;//array_sum(array_column($stats, 'memory')) / $numTests;
        $totalRuntime = 0;//array_sum(array_column($stats, 'time')) / $numTests;

        foreach ($stats as $stat) {
            $totalMemory += $stat['memory'];
            $totalRuntime += $stat['time'];
        }

        $averageMemory = $totalMemory / $numTests;
        $averageRuntime = $totalRuntime / $numTests;

        return compact([
            'numTests',
            'averageMemory',
            'totalMemory',
            'averageRuntime',
            'totalRuntime',
        ]);
    }

    public function run()
    {
        $startTesting = microtime(true);

        $numTests = count($this->testCase);

        if ($numTests < 1) {
            throw new \RuntimeException(
                'You must provide me with some tests for me to run!'
            );
        }

        $maxRuntime = ini_get('max_execution_time');
        $maxRuntime = $maxRuntime ? $maxRuntime - 5 : 10;

        $keys = array_keys($this->testCase);
        $this->runtimeStatistics = array_fill_keys($keys, []);

        $testStartTime = time();
        $lastTick = $testStartTime;

        echo "Beginning tests. Tests may run up to $maxRuntime seconds.\n";

        for (;;) {
            // Tick 10 times, but no faster than every second.
            $tick = max(floor($maxRuntime / 10), 30);
            $currentTime = time();

            if ($currentTime > ($testStartTime + $maxRuntime)) {
                break;
            }

            if ($currentTime > ($lastTick + $tick)) {
                $lastTick = $currentTime;
                echo '.';
            }

            foreach ($this->testCase as $name => $callback) {
                $params = $this->testData ?: [];
                $stats = [];

                memory_reset_peak_usage();
                $startMemoryUsage = memory_get_peak_usage();

                $startTime = hrtime(false);

                // Do test 1000x
                for ($loopTest = 0; $loopTest < 1000; $loopTest++) {
                    call_user_func_array($callback, $params);
                }

                $endTime = hrtime(false);

                $endMemoryUsage = memory_get_peak_usage();

                /**
                 * Calculate the amount of time that has passed. If end - start
                 * is negative, then we have started the next second and should
                 * add this on to our calculation.
                 */
                $s = $endTime[0] - $startTime[0];
                $ns = $endTime[1] - $startTime[1];
                $timeSpent = (pow(10, 9) * $s) + $ns;

                $this->runtimeStatistics[$name][] = [
                    'memory' => $endMemoryUsage - $startMemoryUsage,
                    'time' => $timeSpent,
                ];
            }
        }

        $testTime = microtime(true) - $startTesting;

        $this->output($testTime);
    }

    private static function nanosecs2hr($ns)
    {
        $units = [
            [
                'unit' => 's',
                'power' => pow(10, 9),
            ],
            [
                'unit' => 'ms',
                'power' => pow(10, 6),
            ],
            [
                'unit' => 'us',
                'power' => pow(10, 3),
            ],
            [
                'unit' => 'ns',
                'power' => 1,
            ],
        ];

        foreach ($units as $unit) {
            if ($ns < $unit['power']) {
                continue;
            }

            $result = $ns / $unit['power'];
            return strval(round($result, 2)) . $unit['unit'];
        }

        return $ns;
    }

    private static function bytes2hr($bytes)
    {
        $bytes = floatval($bytes);
        $units = [
            [
                'unit' => 'TB',
                'power' => pow(1024, 4)
            ],
            [
                'unit' => 'GB',
                'power' => pow(1024, 3)
            ],
            [
                'unit' => 'MB',
                'power' => pow(1024, 2)
            ],
            [
                'unit' => 'KB',
                'power' => 1024
            ],
            [
                'unit' => 'B',
                'power' => 1
            ],
        ];

        foreach ($units as $unit) {
            if ($bytes < $unit['power']) {
                continue;
            }

            $result = $bytes / $unit['power'];
            return strval(round($result, 2)). $unit['unit'];
        }
    }
}