CLEVR_eval.py

from __future__ import print_function
import argparse
import json


parser = argparse.ArgumentParser()
parser.add_argument('--questions_file', required=True)
parser.add_argument('--answers_file', required=True)


def main(args):
  # Load true answers from questions file
  true_answers = []
  with open(args.questions_file, 'r') as f:
    questions = json.load(f)['questions']
    for q in questions:
      true_answers.append(q['answer'])

  # Load predicted answers
  predicted_answers = []
  with open(args.answers_file, 'r') as f:
    for line in f:
      predicted_answers.append(line.strip())

  num_true, num_pred = len(true_answers), len(predicted_answers)
  assert num_true == num_pred, 'Expected %d answers but got %d' % (
      num_true, num_pred)

  num_correct = 0
  for true_answer, predicted_answer in zip(true_answers, predicted_answers):
    if true_answer == predicted_answer:
      num_correct += 1

  accuracy = 100.0 * float(num_correct) / len(true_answers)
  print('Got %d / %d correct' % (num_correct, len(true_answers)))
  print('Accuracy = %.4f' % accuracy)



if __name__ == '__main__':
  args = parser.parse_args()
  main(args)