stability_test_all_splits.py

data_collections = tru.get_data_collections()
for dc in data_collections:
    tru.set_data_collection(dc)
    splits = tru.get_data_splits()
    for split_key in splits:
        split = f'{split_key}'
        tru.tester.add_stability_test(
                comparison_data_split_name = split,
                base_data_split_name = 'train',
                metric = 'DIFFERENCE_OF_MEAN',
                warn_if_outside=[-.05, .05],
                fail_if_outside=[-.1, 1],
                overwrite = True
            )