/index.html

## index.html
<!DOCTYPE html>
<html>
<head>
<title>Foo</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0" />
<style type='text/css'>
body {
  background:#000;
  color:#fff;
  font-family:'Georgia';
  margin:0;
}

@-webkit-keyframes blinker {
  from { opacity: 1.0; }
  to { opacity: 0.0; }
}

em {
  -webkit-animation-name: blinker;
  -webkit-animation-iteration-count: infinite;
  -webkit-animation-timing-function: cubic-bezier(1.0,0,0,1.0);
  -webkit-animation-duration: 800ms;
}

strong {
  font-weight:normal;
  color:#FFF707;
}

a {
  color:#FFF707;
  text-decoration:none;
}

</style>
<script type='text/javascript'>
window.onload = function() {
    var s = document.getElementsByTagName('div'), cur = 0;
    if (!s) return;
    function go(n) {
        cur = n;
        var i = 1e3, e = s[n];
        for (var k = 0; k < s.length; k++) s[k].style.display = 'none';
        e.style.display = 'inline-block';
        e.style.fontSize = i + 'px';
        if (e.firstChild.nodeName === 'IMG') {
            document.body.style.backgroundImage = 'url(' + e.firstChild.src + ')';
            e.firstChild.style.display = 'none';
        } else {
            document.body.style.backgroundImage = '';
            document.body.style.backgroundColor = e.style.backgroundColor;
        }
        while (
            e.offsetWidth > window.innerWidth ||
            e.offsetHeight > window.innerHeight) {
            e.style.fontSize = (i -= 10) + 'px';
            if (i < 0) break;
        }
        e.style.marginTop = ((window.innerHeight - e.offsetHeight) / 2) + 'px';
        if (window.location.hash !== n) window.location.hash = n;
        document.title = e.textContent || e.innerText;
    }
    document.onclick = function() {
        go(++cur % (s.length));
    };
    document.onkeydown = function(e) {
        (e.which === 39) && go(Math.min(s.length - 1, ++cur));
        (e.which === 37) && go(Math.max(0, --cur));
    };
    function parse_hash() {
        return Math.max(Math.min(
            s.length - 1,
            parseInt(window.location.hash.substring(1), 10)), 0);
    }
    if (window.location.hash) cur = parse_hash() || cur;
    window.onhashchange = function() {
        var c = parse_hash();
        if (c !== cur) go(c);
    };

    go(cur);
};
</script></head><body>
<div><h1>A/B testing at Good Eggs</h1>
<p>Bob Zoller <a href="&#x6d;&#97;&#x69;&#108;&#116;&#111;&#x3a;&#x62;&#x6f;&#x62;&#64;&#103;&#111;&#111;&#100;&#x65;&#x67;&#103;&#115;&#x2e;&#x63;&#x6f;&#x6d;">&#x62;&#x6f;&#x62;&#64;&#103;&#111;&#111;&#100;&#x65;&#x67;&#103;&#115;&#x2e;&#x63;&#x6f;&#x6d;</a></p>
</div>
<div><h2><a href="http://en.wikipedia.org/wiki/Multi-armed_bandit">Multi-Armed Bandit</a> is cool.</h2>
<p>Simultaneously gather new data and optimize decisions based on existing data.</p>
<ul>
<li><a href="http://genetify.com/demo/">Genetify</a></li>
<li><a href="https://github.com/maccman/abba">Abba</a></li>
</ul>
</div>
<div><h2>Learnings</h2>
<ul>
<li>Genetify is a super hacky POC written in the worst kind of PHP</li>
<li>Multi-armed bandit requires an extra roundtrip to the server</li>
<li>Abba doesn&#39;t support unique users</li>
<li>both systems exist in a vaccuum</li>
</ul>
</div>
<div><h2>Where we landed</h2>
<p>A custom setup with:</p>
<ul>
<li>Abba-like client API</li>
<li>choices stored in LocalStorage</li>
<li>asynchronously registered as a MixPanel super property</li>
<li>MixPanel as a reporting interface</li>
</ul>
</div>
<div><h2>Usage</h2>
<pre><code class="lang-coffeescript">experiment = metrics.experiment(<span class="comment">'1-2-3 header')</span>
  .variant(<span class="comment">'product promotion', 50)</span>
  .variant(<span class="comment">'1-2-3', 50)</span>
  .start()

switch experiment.chosen.name
  when <span class="comment">'product promotion'</span>
    # <span class="keyword">do</span> something
  when <span class="comment">'1-2-3'</span>
    # <span class="keyword">do</span> something <span class="keyword">else</span></code></pre>
</div>
<div><h2>TODO</h2>
<ul>
<li>server-side experiments</li>
<li>multi-armed bandit support</li>
</ul>
</div>
<div><h2>Who&#39;s next?</h2>
<ul>
<li>Alex: Chrome profiler</li>
<li>Brian/Adam: Kale preso</li>
<li>Max: DTrace</li>
</ul>
</div>

## slides.md

      
    Raw
  

              slides.md
            
          
    A/B testing at Good Eggs

Bob Zoller bob@goodeggs.com

Multi-Armed Bandit is cool.

Simultaneously gather new data and optimize decisions based on existing data.

Genetify
Abba


Learnings


Genetify is a super hacky POC written in the worst kind of PHP
Multi-armed bandit requires an extra roundtrip to the server
Abba doesn't support unique users
both systems exist in a vaccuum


Where we landed

A custom setup with:

Abba-like client API
choices stored in LocalStorage
asynchronously registered as a MixPanel super property
MixPanel as a reporting interface


Usage

experiment = metrics.experiment('1-2-3 header')
  .variant('product promotion', 50)
  .variant('1-2-3', 50)
  .start()
  
switch experiment.chosen.name
  when 'product promotion'
    # do something
  when '1-2-3'
    # do something else

TODO


server-side experiments
multi-armed bandit support


Who's next?


Alex: Chrome profiler
Brian/Adam: Kale preso
Max: DTrace
	<!DOCTYPE html>
	<html>
	<head>
	<title>Foo</title>
	<meta charset="utf-8" />
	<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0" />
	<style type='text/css'>
	body {
	background:#000;
	color:#fff;
	font-family:'Georgia';
	margin:0;
	}

	@-webkit-keyframes blinker {
	from { opacity: 1.0; }
	to { opacity: 0.0; }
	}

	em {
	-webkit-animation-name: blinker;
	-webkit-animation-iteration-count: infinite;
	-webkit-animation-timing-function: cubic-bezier(1.0,0,0,1.0);
	-webkit-animation-duration: 800ms;
	}

	strong {
	font-weight:normal;
	color:#FFF707;
	}

	a {
	color:#FFF707;
	text-decoration:none;
	}

	</style>
	<script type='text/javascript'>
	window.onload = function() {
	var s = document.getElementsByTagName('div'), cur = 0;
	if (!s) return;
	function go(n) {
	cur = n;
	var i = 1e3, e = s[n];
	for (var k = 0; k < s.length; k++) s[k].style.display = 'none';
	e.style.display = 'inline-block';
	e.style.fontSize = i + 'px';
	if (e.firstChild.nodeName === 'IMG') {
	document.body.style.backgroundImage = 'url(' + e.firstChild.src + ')';
	e.firstChild.style.display = 'none';
	} else {
	document.body.style.backgroundImage = '';
	document.body.style.backgroundColor = e.style.backgroundColor;
	}
	while (
	e.offsetWidth > window.innerWidth \|\|
	e.offsetHeight > window.innerHeight) {
	e.style.fontSize = (i -= 10) + 'px';
	if (i < 0) break;
	}
	e.style.marginTop = ((window.innerHeight - e.offsetHeight) / 2) + 'px';
	if (window.location.hash !== n) window.location.hash = n;
	document.title = e.textContent \|\| e.innerText;
	}
	document.onclick = function() {
	go(++cur % (s.length));
	};
	document.onkeydown = function(e) {
	(e.which === 39) && go(Math.min(s.length - 1, ++cur));
	(e.which === 37) && go(Math.max(0, --cur));
	};
	function parse_hash() {
	return Math.max(Math.min(
	s.length - 1,
	parseInt(window.location.hash.substring(1), 10)), 0);
	}
	if (window.location.hash) cur = parse_hash() \|\| cur;
	window.onhashchange = function() {
	var c = parse_hash();
	if (c !== cur) go(c);
	};

	go(cur);
	};
	</script></head><body>
	<div><h1>A/B testing at Good Eggs</h1>
	<p>Bob Zoller <a href="mailto:bob@goodeggs.com">bob@goodeggs.com</a></p>
	</div>
	<div><h2><a href="http://en.wikipedia.org/wiki/Multi-armed_bandit">Multi-Armed Bandit</a> is cool.</h2>
	<p>Simultaneously gather new data and optimize decisions based on existing data.</p>
	<ul>
	<li><a href="http://genetify.com/demo/">Genetify</a></li>
	<li><a href="https://github.com/maccman/abba">Abba</a></li>
	</ul>
	</div>
	<div><h2>Learnings</h2>
	<ul>
	<li>Genetify is a super hacky POC written in the worst kind of PHP</li>
	<li>Multi-armed bandit requires an extra roundtrip to the server</li>
	<li>Abba doesn't support unique users</li>
	<li>both systems exist in a vaccuum</li>
	</ul>
	</div>
	<div><h2>Where we landed</h2>
	<p>A custom setup with:</p>
	<ul>
	<li>Abba-like client API</li>
	<li>choices stored in LocalStorage</li>
	<li>asynchronously registered as a MixPanel super property</li>
	<li>MixPanel as a reporting interface</li>
	</ul>
	</div>
	<div><h2>Usage</h2>
	<pre><code class="lang-coffeescript">experiment = metrics.experiment(<span class="comment">'1-2-3 header')</span>
	.variant(<span class="comment">'product promotion', 50)</span>
	.variant(<span class="comment">'1-2-3', 50)</span>
	.start()

	switch experiment.chosen.name
	when <span class="comment">'product promotion'</span>
	# <span class="keyword">do</span> something
	when <span class="comment">'1-2-3'</span>
	# <span class="keyword">do</span> something <span class="keyword">else</span></code></pre>
	</div>
	<div><h2>TODO</h2>
	<ul>
	<li>server-side experiments</li>
	<li>multi-armed bandit support</li>
	</ul>
	</div>
	<div><h2>Who's next?</h2>
	<ul>
	<li>Alex: Chrome profiler</li>
	<li>Brian/Adam: Kale preso</li>
	<li>Max: DTrace</li>
	</ul>
	</div>