Jury BLEU error on single hypothesis and single reference pairs reproduction

jury_single_hypo_ref_pair_error_reproduction.py

from pprint import pprint
from jury import Jury
from jury.metrics import load_metric
METRICS = [
    load_metric("bleu", compute_kwargs={"max_order": 1})
]
outputmetricname = 'bleu_1'
scorer = Jury(metrics=METRICS, run_concurrent=True)

ref = ["For what film was an actor in Alligator nominated for an Academy Award?"]
hyp = ["What film did Michael Vincenzo Gazzo star in?"]

print(' = Single Instance of Hypo-Ref pair = ')
print('\nSingle Hypo - Single Ref pair:')
pprint(scorer(predictions=[hyp], references=[ref])[outputmetricname])
print('\nSingle Hypo - Double Ref pair:')
pprint(scorer(predictions=[hyp], references=[ref*2])[outputmetricname])
print('\nDouble Hypo - Single Ref pair:')
pprint(scorer(predictions=[hyp*2], references=[ref])[outputmetricname])
print('\nDouble Hypo - Double Ref pair:')
pprint(scorer(predictions=[hyp*2], references=[ref*2])[outputmetricname])
print()

ref2 = ["who started ww2 and how did it start"]
hyp2 = ["Who started the war?"]

print(' = Multiple Instances of Hypo-Ref pairs = ')
print('\nSingle Hypo - Single Ref pairs:')
pprint(scorer(predictions=[hyp,hyp2], references=[ref,ref2])[outputmetricname])
print('\nSingle Hypo - Double Ref pairs:')
pprint(scorer(predictions=[hyp,hyp2], references=[ref*2,ref2*2])[outputmetricname])
print('\nDouble Hypo - Single Ref pairs:')
pprint(scorer(predictions=[hyp*2,hyp2*2], references=[ref,ref2])[outputmetricname])
print('\nDouble Hypo - Double Ref pairs:')
pprint(scorer(predictions=[hyp*2,hyp2*2], references=[ref*2,ref2*2])[outputmetricname])
print()