[CORNAC] Discovered a strange behavior on ranking metrics

experiment.py

import time

import cornac
from cornac.models import PMF, NMF, SVD, UserKNN, BPR
from cornac.eval_methods import RatioSplit

from sklearn.metrics import precision_score, recall_score, f1_score
from tabulate import tabulate

SEED = 731926
K = 50
MAX_ITER = 100

models = [
    PMF(k=K, max_iter=MAX_ITER, seed=SEED),
    NMF(k=K, max_iter=MAX_ITER, seed=SEED),
    SVD(k=K, max_iter=MAX_ITER, seed=SEED),
    UserKNN(k=20, seed=SEED, verbose=False)
]

metrics = [
    cornac.metrics.Precision(),
    cornac.metrics.Recall(),
    cornac.metrics.FMeasure()
]

ml_100k = cornac.datasets.movielens.load_feedback(variant="100K")
ratio_split = cornac.eval_methods.RatioSplit(data=ml_100k, test_size=0.2, rating_threshold=4.0, seed=SEED)

def cornac_eval():
    cornac.Experiment(ratio_split, models, metrics, verbose=True).run()

def custom_eval():

    results = []
    for model in models:

        t0 = time.perf_counter()
        model = model.fit(ratio_split.train_set)
        t1 = time.perf_counter()
        train_time = t1 - t0

        predictions = {}
        for u, i, r in zip(*ratio_split.test_set.uir_tuple):
            if u not in predictions: 
                predictions[u] = []
            predictions[u].append((u, i, r, model.rate(u, i)))

        precision, recall, f1 = 0, 0, 0
        for uid in predictions:
            u_true_r, u_est_r = [], []
            for u, _, true_r, est_r in predictions[uid]:
                u_true_r.append(0 if true_r <= ratio_split.rating_threshold else 1)
                u_est_r.append(0 if est_r <= ratio_split.rating_threshold else 1)

            precision += precision_score(u_true_r, u_est_r, pos_label=1, zero_division=0)
            recall += recall_score(u_true_r, u_est_r, pos_label=1, zero_division=0)
            f1 += f1_score(u_true_r, u_est_r, pos_label=1, zero_division=0)

        num_test_user = len(predictions.keys())
        precision /= num_test_user
        recall /= num_test_user
        f1 /= num_test_user

        results.append([model.name, f1, precision, recall, train_time])

    print("\nCUSTOM EVALUATION")
    print(tabulate(results, headers=["Model", "F1 score", "Precision", "Recall", "Train (s)"], floatfmt=".4f"))

cornac_eval()
custom_eval()