caliberation_error.py

import pandas as pd
import numpy as np

def calibration_error(y_true, y_prob, n_bins=5, strategy='uniform', return_expected_caliberation_error=True):
    if strategy == 'quantile':  # Determine bin edges by distribution of data
        quantiles = np.linspace(0, 1, n_bins + 1)
        bins = np.percentile(y_prob, quantiles * 100)
        bins[-1] = bins[-1] + 1e-8
    elif strategy == 'uniform':
        bins = np.linspace(0., 1. + 1e-8, n_bins + 1)
    else:
        raise ValueError("Invalid entry to 'strategy' input. Strategy "
                            "must be either 'quantile' or 'uniform'.")

    y_prob_max = np.max(y_prob, axis=-1)
    binids = np.digitize(y_prob_max, bins) - 1

    y_correct_classified = (np.argmax(y_true, axis=-1) == np.argmax(y_prob, axis=-1)).astype(int) 
    


    bin_sums = np.bincount(binids, weights=y_prob_max, minlength=len(bins))
    bin_true = np.bincount(binids, weights=y_correct_classified, minlength=len(bins))
    bin_total = np.bincount(binids, minlength=len(bins))

    nonzero = bin_total != 0

    # acc(Bm)
    prob_true = bin_true[nonzero] / bin_total[nonzero]

    #conf(Bm)
    prob_pred = bin_sums[nonzero] / bin_total[nonzero]

    expected_caliberation_error = np.sum(bin_total[nonzero] * np.abs(prob_true - prob_pred))/bin_total[nonzero].sum()
    overconfidence_error = np.sum(bin_total[nonzero] *  prob_pred * np.max(np.concatenate(((prob_pred - prob_true).reshape(-1, 1), np.zeros((1, len(prob_pred))).T), axis=1), axis=-1)/bin_total[nonzero].sum())
    return prob_true, prob_pred, expected_caliberation_error, overconfidence_error