README.md

Erudition

Some short sentence describing what this esoterically named lib does.

Goals

• Wrap two read paths and measure corectness and execution time.
• Integrated feature flag to always use control (existing code).
• Automatic counts and timings collection.
  • erudition.experiment.<name>

Usages

Where using a new static method on same class:

$controlResult = ParallelCodePath::execute('experiment.name', function(ParallelCodePath $instance) {
    $instance->control(['\QDateTime', 'getTimestamp']);
    $instance->candidate(['\QDateTime', 'someNewAwesomeGetTimestamp']);

    $instance->comparator(function($resultA, $resultB) {
        return $resultA === $resultB;
    });
});

Where replacing a function call with a new method call, or similar:

// old
_sp('somg language string');

// new
$someValue = 'some value';
$lm = $pimple[LanguageManager::class];
$lm->get('lang.string.name', ['replacement' => $someValue]);

// ParallelCodePath controlled:
$result = ParallelCodePath::execute('experiment.name', function(ParallelCodePath $instance) use($someValue) {
    $instance->control(
        '_sp',
        ['somg language string']
    );
    $instance->candidate(
        [$pimple[LanguageManager::class], 'get'],
        ['replacement' => $someValue]
    );

    $instance->comparator(function($resultA, $resultB) {
        return $resultA === $resultB;
    });
});

Where replacing a method call on with a new method on an instance:

$this->createdAt = new \QDateTime();

$object = $this->createdAt;
$result = ParallelCodePath::execute('experiment.name', function(ParallelCodePath $instance) use($object) {
    $instance->control([$this->createdAt, 'getTimestamp']);
    $instance->candidate([$this->createdAt, 'awesomeGetTimestamp']);

    $instance->comparator(function($resultA, $resultB) {
        return $resultA == $resultB;
    });
});

Instantiating an instance of ParallelCodePath:

$instance = new \QDateTime();

$test = new ParallelCodePath('test', null);
$test->control([$instance, 'getTimestamp']);
$test->candidate([$instance, 'awesomeGetTimestamp']);

// optional comparator, uses phpunits compare library by default.
$test->comparator(function($resultA, $resultB) {
    return $resultA === $resultB;
});
$result = $test->run('test');

ParallelCodePath.php

<?php
namespace Erudition;

use RuntimeException;
use SebastianBergmann\Comparator\Factory;
use SebastianBergmann\Comparator\ComparisonFailure;
use Throwable;


class ParallelCodePath
{

    protected $control;
    protected $controlParams = array();

    protected $candidate;
    protected $candidateParams = array();

    protected $comparator = null;

    /**
     * @param $experimentName
     * @param $callable
     * @return Result
     * @throws Throwable
     */
    public static function execute($experimentName, $callable)
    {
        $instance = new self;
        call_user_func_array($callable, array($instance));
        return $instance->run($experimentName);
    }

    /**
     * @param callable $callable
     * @param null $params
     * @return $this
     */
    public function control(callable $callable, $params = null)
    {
        $this->control       = $callable;
        $this->controlParams = $params;

        return $this;
    }

    /**
     * @param callable $callable
     * @param null $params
     * @return $this
     */
    public function candidate(callable $callable, $params = null)
    {
        $this->candidate       = $callable;
        $this->candidateParams = $params;

        return $this;
    }

    /**
     * @param callable $callable
     * @return $this
     */
    public function comparator(callable $callable)
    {
        $this->comparator = $callable;
        return $this;
    }

    /**
     * Runs the control and candidate code, returning the control results.
     * Note: Throws a runtime exception if either codepath is missing.
     *
     * @param string $experimentName
     * @return Result
     * @throws Throwable
     */
    public function run($experimentName)
    {
        if (empty($this->control) || empty($this->candidate)) {
            throw new RuntimeException('Missing control or candidate callable');
        }

        // randomise order in which the two paths are called.
        if (random_int(0, 1) === 1) {
            $controlResult   = $this->observeCallable($experimentName, 'control', $this->control, $this->controlParams);
            $candidateResult = $this->observeCallable($experimentName, 'candidate', $this->candidate, $this->candidateParams);
        } else {
            $candidateResult = $this->observeCallable($experimentName, 'candidate', $this->candidate, $this->candidateParams);
            $controlResult   = $this->observeCallable($experimentName, 'control', $this->control, $this->controlParams);
        }

        $comparisonResult = $this->compare($controlResult->getResult(), $candidateResult->getResult());

        $this->logResults($controlResult, $comparisonResult);
        
        if ($controlResult->isException()) {
            throw $controlResult->getException();
        }

        return $controlResult->getValue();
    }

    /**
     * @param $experimentName
     * @param $trial
     * @param  $callable
     * @param array $params
     * @return Result
     */
    protected function observeCallable($experimentName, $trial, &$callable, $params)
    {
        $result = $exception = null;

        $start = microtime(true);
        try {
            $result = call_user_func_array($callable, $params);
        } catch(\Exception $e) {
            $exception = $e;
        }
        $duration = microtime(true) - $start;

        return new Result($experimentName, $trial, $result, $duration, $exception);
    }

    /**
     * @param  Result $a
     * @param  Result $b
     * @return bool
     */
    protected function compare($a, $b)
    {
        if (is_callable($this->comparator)) {
            return call_user_func_array($this->comparator, array($a->getResult(), $b->getResult()));
        }

        return $this->defaultCompare($a->getResult(), $b->getResult());
    }

    /**
     * Defaults compare function using assertEquals from PHPUnit.
     * @todo: Interface and extract this, make dependency optional requirement in composer.json.
     * @todo is this slow or a terrible idea, just assert ===?
     *
     * @param  mixed $a
     * @param  mixed $b
     * @return bool
     */
    protected function defaultCompare($a, $b)
    {
        // TODO(philipmuir): move compare functions to own interface and have phpunit comparisions as an option
//        $factory = new Factory;
//        $comparator = $factory->getComparatorFor($a, $b);
//
//        try {
//            return $comparator->assertEquals($a, $b);
//        }  catch (ComparisonFailure $failure) {
//            return false;
//        }
        return $a == $b;
    }

    protected function logResults($controlResult, $candidateResult)
    {
        //todo: syslog errors/statsd counts
    }
}

Result.php

<?php
namespace Erudition;

use Throwable;

class Result
{
    /** @var  mixed */
    protected $value;

    /** @var  float */
    protected $duration;

    /** @var  null|Throwable */
    protected $exception;

    /** @var string */
    private $experimentName;

    /** @var string */
    private $trialName;

    /**
     * Result constructor.
     * @param string $experimentName
     * @param string $trialName
     * @param $value
     * @param float $duration
     * @param Throwable $exception
     */
    public function __construct(
        string $experimentName,
        string $trialName,
        $value,
        float $duration,
        Throwable $exception = null)
    {
        $this->experimentName = $experimentName;
        $this->trialName = $trialName;
        $this->value = $value;
        $this->duration = $duration;
        $this->exception = $exception;
    }

    /**
     * Returns the exception if set, otherwise the value.
     * @return Throwable|mixed|null
     */
    public function getResult()
    {
        if ($this->isException()) {
            return $this->getException();
        }

        return $this->getValue();
    }

    /**
     * @return mixed
     */
    public function getValue()
    {
        return $this->value;
    }

    /**
     * @return int
     */
    public function getDuration()
    {
        return $this->duration;
    }

    /**
     * @return Throwable|null
     */
    public function getException()
    {
        return $this->exception;
    }

    /**
     * @return bool
     */
    public function isException()
    {
        return ($this->exception instanceof Throwable);
    }
}