calibration_hist.py

import numpy as np
import gjp
# https://github.com/niplav/iqisa/blob/master/iqisa.py
import iqisa as iqs
import matplotlib.pyplot as plt

df=gjp.load_markets()

df["isCorrect"] = (df.outcome == df.answer_option).astype(float)
fig = plt.figure(figsize=(9, 9))


def calibration_histogram(predicted_probs, actual_probs, n_bins=10):
    # Compute the bin edges and centers
    bin_edges = np.linspace(0, 1, n_bins+1)
    bin_centers = (bin_edges[1:] + bin_edges[:-1]) / 2
    
    # Compute the number of samples and total actual probability in each bin
    actual_probs_binned, _ = np.histogram(predicted_probs, bins=bin_edges, weights=actual_probs)
    
    # Compute the mean actual probability in each bin (avoiding division by zero)
    n_samples, _ = np.histogram(predicted_probs, bins=bin_edges)
    mean_actual_probs = np.divide(actual_probs_binned, n_samples, out=np.zeros_like(actual_probs_binned), where=n_samples!=0)
    
    # Plot the calibration histogram
    fig, ax = plt.subplots()
    ax.plot([0, 1], [0, 1], linestyle='--', color='gray')
    ax.bar(bin_centers, mean_actual_probs)#, align='edge', width = 0.5)
    ax.set_xlim([0, 1])
    ax.set_ylim([0, 1])
    ax.set_xlabel('Predicted Probability')
    ax.set_ylabel('Actual Probability')
    
    return fig, ax

calibration_histogram(df.probability, df.isCorrect, n_bins=10)
plt.show()