Comparison between Empirical Bernstein, Hoeffding and CLT

empbern.py

"""
Comparison of empirical bernstein measures

See https://el-hult.github.io/2022/03/18/empirical-bernstein-bounds.html

"""
import numpy as np
import matplotlib.pyplot as plt
import scipy.stats as sps
import pandas as pd
import seaborn as sns

plt.rcParams.update({"font.size": 8, "legend.fontsize": 8, "figure.figsize": (3, 2)})

sns.set_theme()
ress = []

for n in np.logspace(1,3):
    for delta in [0.01, 0.05, 0.1]:  # confidence level
        for V_n in [0.01, 0.1, 0.25]:  # observed variance

            x = np.log(3 / delta)
            audibert = np.sqrt(2 * V_n * x / n) + 3 * x / n

            z = sps.norm.ppf(1 - delta / 2)
            clt = z * np.sqrt(V_n / n)

            x = np.log(2 / delta)
            hoeffding = np.sqrt(x / 2 / n)

            ress.append(dict(name="clt",ci_len=clt,delta=delta,V_n=V_n,n=n))
            ress.append(dict(name="audibert",ci_len=audibert,delta=delta,V_n=V_n,n=n))
            ress.append(dict(name="hoeffding",ci_len=hoeffding,delta=delta,V_n=V_n,n=n))

df = pd.DataFrame(ress)
df.reset_index(inplace=True)
sns.relplot(
    x="n",
    y="ci_len",
    hue="name",
    data=df,
    col="delta",
    row="V_n",
    facet_kws={"subplot_kws": {"xscale": "log", "yscale": "log"}},
)
plt.show()