bertscore

!pip -q install nlp
!pip -q install bert_score

from nlp import load_metric

metric = load_metric("bertscore")



import numpy as np
logging.getLogger("transformers").setLevel(logging.ERROR)

model.eval()

refrences = []
predictions = []
dataset = test_seen_dataset

f_scores = []
for i in tqdm(range(len(dataset))):
  kwargs = {#'num_beams':1,
            'num_return_sequences':1,'temperature':1, 'max_length':50, 'early_stopping':True,
            'no_repeat_ngram_size':3,
            'do_sample':True
            }
  #kwargs = {}
  hk_pair =  dataset[i]['input_pair'].to(dev)
  hk_segment = dataset[i]['input_pair_segments'].to(dev)
  response = dataset[i]['response'].to(dev)
  generateds = model.generate(hk_pair, hk_segment, **kwargs)[0]
  prediction = dec_tokenizer.decode(generateds, skip_special_tokens=True)
  refrence = dec_tokenizer.decode(dataset[i]['response'], skip_special_tokens=True)
  refrences.append(refrence)
  predictions.append(prediction)


res = metric.compute(predictions=predictions, references=refrences,
               model_type='bert-base-uncased',device='cuda:0',
               rescale_with_baseline=False, lang='en'
               )
res['f1'].mean()


f_scores = []
for i in range(len(refrences)):
  refrence = np.array(enc_tokenizer.encode(refrences[i]))
  candid = np.array(dec_tokenizer.encode(predictions[i]))
  intersections = np.intersect1d(refrence, candid)
  recall = len(intersections) / len(refrence)
  precision = len(intersections) / len(candid)
  f1_score = 2 * (precision * recall) / (precision + recall)
  f_scores.append(f1_score)
print( sum(f_scores) / len(f_scores))