SeGLiR - A/A test

index.js

const jstat = require("jStat");
var glr = require("seglir");

const zip = (a, b) => a.map((k, i) => [k, b[i]]);

// create an instance of a two-sided test comparing bernoulli proportions, with indifference region with size 0.01, alpha-level = 0.05, beta-level = 0.10
// first is test, second is sides, third is indifference, fourth is type1_error, fifth is type 2 error
var alpha = 0.05;
var beta = 0.10;
var indifference = 0.01;

// when addData() returns string 'true' or 'false', the test is concluded
// 'true' means the null hypothesis was accepted
// 'false' means the null hypothesis was rejected (and the alternative hypothesis accepted)

// get bias-adjusted estimates
// test.estimate();

// Allocate N users according to allocation probabilities.
const alloc_probs = [0.5, 0.5];
const conversion_rates = [0.1, 0.1];

const NUM_VISITORS = 100000
const NUM_TRIALS = 100


// Copy pasta'ed thresholds so see what b0 and b1 are.
var thresholds = {}
thresholds['bernoulli'] = {
    'two-sided' : {
        0.05 : {
            0.05 : {
                0.1 : [139.5, 30.5],
            },
            0.10 : {
                0.4 : [68.6, 12.9], 
                0.2 : [98, 14.6], 
                0.1 : [139, 14.5],
                0.05 : [172, 15.5], 
                0.025 : [220, 15.5], 
                0.01 : [255, 15.7]
            },
            0.20 : {
                0.4 : [68, 5.4],
                0.2 : [90.5, 6.5],
                0.1 : [134, 6.9],
                0.05 : [168, 7.3],
                0.025 : [214, 7.4],
                0.01 : [254, 7.5]
            }
        }
    },
    'one-sided' : {
        0.05 : {
            0.05 : {
                0.2 : [39.1, 39.1],
                0.1 : [42, 42],
                0.05 : [70, 70],
                0.025 : [77, 77],
                0.01 : [95,95]
            },
            0.10 : {
                0.2 : [39.1, 18.5],
                0.1 : [41.4, 24.4],
                0.05 : [65, 27.5],
                0.025 : [74.5, 33.8],
                0.01 : [90, 46]
            }
        }
    }
}
var [b0, b1] = thresholds["bernoulli"]["one-sided"][alpha][beta][indifference]
console.log("b0: ", b0, "b1: ", b1)
let num_rejected = 0;
let num_accepted = 0;
var var_total_units;
var var_conversions;
var results;
for (let index = 0; index < NUM_TRIALS; index++) {
    var_total_units = [0, 0];
    var_conversions = [0, 0];

    let test = new glr.test("bernoulli", "one-sided", indifference, alpha, beta);
    let total_visitors = 0;
    while (total_visitors < NUM_VISITORS) {
        // Allocate visitor to variation.
        let val = Math.random();
        let var_index = 0;
        while (val - alloc_probs[var_index] > 0) {
            val -= alloc_probs[var_index]
            var_index++;
        }
        var_total_units[var_index]++;
    
        // Determine whether visitor converts or not.
        // let outcome = d3.randomBernoulli(conversion_rates[var_index])();
        let outcome = Math.random() < conversion_rates[var_index] ? 1 : 0
        var_conversions[var_index] += outcome;
        let addDataResult;
        if (var_index == 0) {
            addDataResult = test.addData({x : outcome});
        } else {
            addDataResult = test.addData({y : outcome});
        }
        results = test.getResults();
        // console.log(results)
        if (results['finished'] == true) {
            // console.log(test.pValue())
            let resultStr = addDataResult == 'false' ? 'null rejected' : 'null accepted'
            console.log(`=====Trial ${index} FINISHED. ${resultStr} =====`)
            console.log(results)
            if (results['L_an'] >= b0) {
                num_rejected+=1
            } 
            if (results['L_bn'] >= b1) {
                num_accepted+=1
            }
            break;
        }
        total_visitors++;
    }
    if (results['finished']) {
        zip(var_conversions, var_total_units).forEach(element => {
            console.log(element, element[0] / element[1])
        });
        console.log(`====TRIAL FINISHED END====`)
    }
}

console.log("% rejected: ", num_rejected / NUM_TRIALS * 100)
console.log("% accepted: ", num_accepted / NUM_TRIALS * 100)

// Calculate chi-square test statistic for allocations