relative_performance_test.py

data_collections = tru.get_data_collections()
for dc in data_collections:
    tru.set_data_collection(dc)
    data_splits = ["train","2010","2011","2012","2013","2014","2015","2016"]
    ref_model = "linear_" + dc
    for split in data_splits:
        tru.tester.add_performance_test(
            data_split_name = split,
            metric = 'AUC',
            warn_threshold_type = "ABSOLUTE", #this is default, specifying for clarity
            warn_if_less_than = 0.80, #treshold is reference model perf + value
            fail_threshold_type = "RELATIVE",
            fail_if_less_than = 0,
            reference_model_name = ref_model #set model to test against
        )