zscore.rb

  def z_score(variation_id)
    # Taken from https://github.com/andrew/split/blob/master/lib/split/zscore.rb
    n_1 = views_count(variation_id).to_f
    n_2 = views_count(popup_id).to_f
    p_1 = conversion_rate(variation_id)
    p_2 = conversion_rate(popup_id)

    if n_1 < 30 || n_2 < 30 || (p_1 * n_1 < 5) || (p_2 * n_2 < 5)
      return false
    end

    diff = (p_2 - p_1).abs

    # if the difference is bigger than 100%, return the winner
    return 3.0 if diff >= 1

    # normalize if ctr is bigger than 100%
    if p_1 > 1 or p_2 > 1
      normalize_factor = [ p_1, p_2 ].max - 1.0
      p_1 -= normalize_factor
      p_2 -= normalize_factor
    end

    # Formula for standard error: root(pq/n) = root(p(1-p)/n)
    s_1 = Math.sqrt p_1 * (1-p_1) / n_1
    s_2 = Math.sqrt p_2 * (1-p_2) / n_2

    # Formula for pooled error of the difference of the means: root(π*(1-π)*(1/na+1/nc)
    # π = (xa + xc) / (na + nc)
    pi = (p_1 * n_1 + p_2 * n_2) / (n_1 + n_2)
    s_p = Math.sqrt pi * (1-pi) * (1 / n_1 + 1 / n_2)

    # Formula for unpooled error of the difference of the means: root(sa**2/na + sc**2/nc)
    s_unp = Math.sqrt((s_1 ** 2) + (s_2 ** 2))

    # Boolean variable decides whether we can pool our variances
    pooled = s_1 / s_2 < 2 && s_2 / s_1 < 2

    # Assign standard error either the pooled or unpooled variance
    se = pooled ? s_p : s_unp

    # Calculate z-score
    ((p_1 - p_2) / se).abs

  rescue Math::DomainError
    Rails.logger.debug("SplitTest::z_score: #{variation_id} | #{p_1} | #{p_2} | #{s_1} | #{s_2}")
    false
  end