ProbabilityHelpers

ProbabilityHelpers.scala

package com.celtra.experimentation

import org.apache.commons.math3.distribution.{BetaDistribution,NormalDistribution}
import org.apache.commons.math3.random.Well19937c

object ProbabilityHelpers {
  val PRIOR_ALPHA = 1
  val PRIOR_BETA = 1
  val DEFAULTNSIM = 10000

  def computeBestVariantBeta(variantMetrics: Map[String, (Long, Long)], seed: Long = 0, NSim: Int = DEFAULTNSIM): Map[String, Double] = {
    // Same Random Generator Used by All the Variants
    val rng = new Well19937c(seed)

    // Assumes Uninformative prior
    val variantDraws = variantMetrics.mapValues{ case (successes, failures) =>
      val beta = new BetaDistribution(rng, successes + PRIOR_ALPHA, failures + PRIOR_BETA)
      beta.sample(NSim)
    }

    tallyBestVariant(variantDraws.toIndexedSeq, NSim)
  }

  // Assume Posterior Distribution is 100% determined by Data. (no Prior)
  def computeBestVariantNormal(variantMetrics: Map[String, (Double, Double)], seed: Long = 0, NSim: Int = DEFAULTNSIM): Map[String, Double] = {
    // Same Random Generator Used by All the Variants
    val rng = new Well19937c(seed)

    val variantDraws = variantMetrics.mapValues{ case (mean, sd) =>
      val normal = new NormalDistribution(rng, mean, sd)
      normal.sample(NSim)
    }

    tallyBestVariant(variantDraws.toIndexedSeq, NSim)
  }

  private def tallyBestVariant(draws: IndexedSeq[(String, Array[Double])], NSim: Int = DEFAULTNSIM): Map[String, Double] = {
    // Getting the Winning Variable Name By Simulation Round
    val results = (0 to NSim-1).map(idx => draws.maxBy(_._2(idx))._1)

    // For each Variant, compute the probability of being the best
    draws.map { case (variantName, _) =>
      val probOfBest = results.filter(_ == variantName).length.toDouble / NSim
      variantName -> probOfBest
    }.toMap
  }
}