check_fy_vs_likelihood.py

# compare FY entmax losses with (log)-likelihood objectives
# author: vlad niculae

import numpy as np
import torch
import matplotlib.pyplot as plt

from entmax import entmax_bisect, entmax_bisect_loss

def main(alpha=1.5):
    plt.figure()
    ts = torch.linspace(-4, 4, 50)

    # since the implementation is for multiclass,
    # we represent the score as z=[0, t] with y_true=[0, 1]
    Z = torch.stack((torch.zeros_like(ts), ts)).t()

    # (however, y_true is stored as indices)
    y_true = torch.ones_like(ts, dtype=torch.long)

    P = entmax_bisect(Z, alpha=alpha)
    plt.title("binary loss for score=t if true_y=1 // alpha={}".format(alpha))
    plt.xlabel("t")
    plt.plot(ts, 1 - P[:, 1], label="1-p")
    plt.plot(ts, -torch.log(P[:, 1]), label="-logp")

    loss = entmax_bisect_loss(Z, y_true, alpha=alpha)
    plt.plot(ts, loss, label="FY loss")
    plt.legend()
    plt.savefig(f"entmax-{alpha}.png")

if __name__ == '__main__':
    main(alpha=1.5)
    main(alpha=2)
    main(alpha=1.05)

Approaching alpha->1, the Tsallis entmax FY loss approaches the negative log likelihood. But for any alpha>1, the nll becomes infinite at a point, while the FY loss is always nice.