Python: LLM-as-judge evaluation harness — rubric scoring with aggregate metrics

llm_judge.py

"""
LLM-as-judge evaluation harness.
Given a list of (input, generated_output, expected_output) tuples, ask a
strong model to score each generation on a rubric and produce aggregate metrics.

Install:
    pip install openai

Env:
    export OPENAI_API_KEY=sk-...
"""

import os
import json
from openai import OpenAI
from dataclasses import dataclass
from statistics import mean

client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
JUDGE_MODEL = "gpt-4o"  # use your strongest model to judge

RUBRIC = """
Score the candidate answer on three axes, each 1-5:

- correctness: Does the answer factually match the reference?
- completeness: Does it cover everything the reference covers?
- clarity: Is it clear, well-structured, free of filler?

Return ONLY valid JSON of the form:
{"correctness": int, "completeness": int, "clarity": int, "reason": "one-sentence justification"}
""".strip()


@dataclass
class Sample:
    input: str
    generated: str
    expected: str


def judge_one(sample: Sample) -> dict:
    prompt = f"""{RUBRIC}

INPUT:
{sample.input}

REFERENCE ANSWER:
{sample.expected}

CANDIDATE ANSWER:
{sample.generated}
"""
    res = client.chat.completions.create(
        model=JUDGE_MODEL,
        response_format={"type": "json_object"},
        messages=[{"role": "user", "content": prompt}],
    )
    raw = res.choices[0].message.content.strip()
    return json.loads(raw)


def evaluate(samples: list[Sample]) -> dict:
    scored = [judge_one(s) for s in samples]
    agg = {
        "n": len(scored),
        "correctness": round(mean(s["correctness"] for s in scored), 2),
        "completeness": round(mean(s["completeness"] for s in scored), 2),
        "clarity": round(mean(s["clarity"] for s in scored), 2),
        "per_sample": scored,
    }
    return agg


if __name__ == "__main__":
    samples = [
        Sample(
            input="Capital of France?",
            generated="Paris is the capital of France.",
            expected="Paris",
        ),
        Sample(
            input="Explain photosynthesis in one sentence.",
            generated="Plants eat sunlight.",
            expected="Photosynthesis is how plants convert sunlight, water, and CO2 into glucose and oxygen.",
        ),
    ]
    results = evaluate(samples)
    print(json.dumps(results, indent=2))