rrbayesian.py

#!/usr/bin/env python
#http://engineering.richrelevance.com/bayesian-ab-tests/
from numpy.random import beta as beta_dist
import numpy as np
N_samp = 10000 # number of samples to draw
clicks_A = 450 # insert your own data here
views_A = 56000
clicks_B = 345 # ditto
views_B = 49000
alpha = 1
beta = 1
A_samples = beta_dist(clicks_A+alpha, views_A-clicks_A+beta, N_samp)
B_samples = beta_dist(clicks_B+alpha, views_B-clicks_B+beta, N_samp)

if __name__ == "__main__":
    # Probability that CTR_a > CTR_b
    print np.mean(A_samples > B_samples)
    # Probability that A relative to B is 3% lift
    print np.mean(100.*(A_samples - B_samples)/B_samples > 3)