eval_loop

eval_loop.py

for step, batch in enumerate(valid_dataloader):
    if config.is_comparison:
        batch = get_model_specific_batch(batch, model_name)

    with torch.no_grad():
        outputs = model(**batch)
    predictions = outputs.logits.argmax(-1)
    predictions, references = accelerator.gather_for_metrics(
        (predictions, batch['labels'])
    )

    f1_metric.add_batch(
        predictions=predictions,
        references=references
    )
    acc_metric.add_batch(
        predictions=predictions,
        references=references
    )

f1_score = f1_metric.compute(average=F1_AVERAGE)['f1']
acc_score = acc_metric.compute()['accuracy']
return acc_score, f1_score