eval

!pip -q install nlp
!pip -q install bert_score

from nlp import load_metric

metric = load_metric("bertscore")



import numpy as np
logging.getLogger("transformers").setLevel(logging.ERROR)

model.eval()



refrences = []
predictions = []

kwargs = {'num_beams':1,
          'num_return_sequences':1,'temperature':1, 'max_length':50,'early_stopping':True,
          'no_repeat_ngram_size':3,
          'decoder_start_token_id':0,
          'eos_token_id':2
          #'do_sample':True
          }
for batch_idx, batch in tqdm(enumerate(test_seen_loader)):
  pair_batch, segment_batch, response_batch = batch
  pair_batch = pair_batch.to(dev)
  segment_batch = segment_batch.to(dev)
  response_batch = response_batch.to(dev)
  generateds = model.seq2seq.generate(pair_batch, **kwargs)
  new_prediction = (dec_tokenizer.batch_decode(generateds, skip_special_tokens=True))
  new_refrence = dec_tokenizer.batch_decode(response_batch, skip_special_tokens=True)
  refrences.extend(new_refrence)
  predictions.extend(new_prediction)
  
  
  res = metric.compute(predictions=predictions, references=refrences,
               model_type='bert-base-uncased',device='cuda:0',
               rescale_with_baseline=False, lang='en'
               )
res['f1'].mean()



f_scores = []
for i in range(len(refrences)):
  refrence = np.array(enc_tokenizer.encode(refrences[i]))
  candid = np.array(dec_tokenizer.encode(predictions[i]))
  intersections = np.intersect1d(refrence, candid)
  recall = len(intersections) / len(refrence)
  precision = len(intersections) / len(candid)
  f1_score = 2 * (precision * recall) / (precision + recall)
  f_scores.append(f1_score)
print( sum(f_scores) / len(f_scores))