test.php

<?php

// Instructions: save this file to the base diretory of the kumite-moa repo..

require 'vendor/autoload.php';

/**
 * Calculates the statistical significance of a boolean experiment over
 * multiple events (e.g. contests), where the result for each event is
 * a true/false outcome (e.g. contest was or was not refunded).
 *
 * The experiment has two conditions, "treatment" and "control": for each
 * we pass the mean occurrence (between 0 and 1), and the number of events
 * which happened. The function returns the likelihood p that any difference
 * seen occurred by chance.
 *
 * If p is small (e.g. < 0.05) we typically say the difference is
 * "significant".
 */
function significanceTestBoolean($pTreatment, $pControl, $nTreatment, $nControl)
{
  # convert to a z score
  $sigmaCombined = sqrt($pTreatment * (1 - $pTreatment) / $nTreatment +
    $pControl * (1 - $pControl) / $nControl);
  # add 1e-8 to denominator to avoid divide by zero
  $z = ($pTreatment - $pControl) / ($sigmaCombined + 1e-8);

  # return the likelihood of a value this extreme or greater under
  # the null hypothesis
  $p = 2 * (1 - stats_cdf_normal(abs($z), 0, 1, 1));

  return $p;
}

class TestCookieAdapter
{
  public function getCookies()
	{
		return array();
	}

	public function setCookie($name, $data)
	{
	}

	public function getCookie($name)
	{
	}
}

class UCB1Allocator implements \Kumite\Allocator
{
	public function __construct($test, $event)
	{
		$this->test = $test;
		$this->event = $event;
	}

	public function allocate($variants)
	{
		$tmp = array();

		foreach ($variants as $v)
		{
			$armCount = KumiteParticipant::count(array(
				'test' => $this->test,
				'variant' => $v,
			));

			if ($armCount == 0)
				return $v;

			$tmp[$v] = list($e, $n) = $this->getConversionRate($v);
			$r = $e / (float)$n;
			$b = $this->getBonus($v);
			printf("%s - (%d / %d) %2.3f + %2.3f = %2.3f   |   ", $v, $e, $n, $r, $b, $r + $b);

			$ucb_values[$v] = $r + $b;
		}

		$pR = $tmp['red'][0] / $tmp['red'][1];
		$pG = $tmp['green'][0] / $tmp['green'][1];

		$p = significanceTestBoolean($pR, $pG, $tmp['red'][1], $tmp['green'][1]);
		printf(" p=%3.3f (%2.0f%%) %s |  ", $p, (1.0 - $p) * 100.0, $p < 0.05 ? '*' : ' ');


		return array_pop(array_keys($ucb_values, max($ucb_values)));
	}

	// --

	private function getConversionRate($variant)
	{
		$e = count(KumiteEvent::distinct('participantId', array(
			'test' => $this->test,
			'variant' => $variant,
			'event' => $this->event
		)));

		$n = KumiteParticipant::count(array(
			'test' => $this->test,
			'variant' => $variant,
		));

		return array($e, $n);
	}

	private function getBonus($variant)
	{
		$totalCount = KumiteParticipant::count(array(
			'test' => $this->test,
		));

		$armCount = KumiteParticipant::count(array(
			'test' => $this->test,
			'variant' => $variant,
		));

		return sqrt((2.0 * log($totalCount)) / $armCount);
	}
}

$config = array(
  'storageAdapter' => new MoaStorageAdapter(),
  'cookieAdapter' => new TestCookieAdapter(),
  'tests' => array(
    'redvsgreen' => array(
      'start' => '2013-01-01',
      'end' => '2014-01-01',
      'default' => 'red',
      'variants' => array(
        'red',
        'green'
      )
    ),
  )
);


Moa::setup(function() {
	$mongo = new Mongo();
	return $mongo->selectDB('mbatest');
});

KumiteEvent::drop();
KumiteParticipant::drop();

for ($i=0; $i<1000000; $i++)
{
	Kumite::setup($config);
	Kumite::start('redvsgreen', new UCB1Allocator('redvsgreen', 'done'));

	$v = Kumite::variant('redvsgreen')->key();
	echo "$v\n";

	if ($v == 'red')
	{
		if (rand(0, 99) < 42)
		{
			Kumite::event('redvsgreen', 'done');
		}
	}
	elseif ($v == 'green')
	{
		if (rand(0, 99) < 40)
		{
			Kumite::event('redvsgreen', 'done');
		}
	}
}