Setup and benchmark script using the official OpenAI Agents SDK

agents_run.py

import subprocess
import asyncio
import time
import json
from pathlib import Path
from agents import Agent, Runner, function_tool

PROJECT_ROOT = Path(__file__).parent

@function_tool
def read_file(path: str) -> str:
    """Read a source file."""
    try:
        p = PROJECT_ROOT / path
        if not p.exists():
            return f"NOT FOUND: {path}"
        if p.stat().st_size > 40_000:
            return f"TOO LARGE: {path}"
        return p.read_text()
    except Exception as e:
        return f"ERROR: {e}"

@function_tool
def write_file(path: str, content: str) -> str:
    """Write content to a file."""
    try:
        p = PROJECT_ROOT / path
        p.parent.mkdir(parents=True, exist_ok=True)
        p.write_text(content)
        return f"OK: {path}"
    except Exception as e:
        return f"ERROR: {e}"

@function_tool
def list_files(directory: str = "src") -> str:
    """List source files, excluding node_modules and build artifacts."""
    try:
        p = PROJECT_ROOT / directory
        if not p.exists():
            return f"NOT FOUND: {directory}"
        excluded = {"node_modules", ".git", "__pycache__", "dist", "build", "venv"}
        results = [
            str(f.relative_to(PROJECT_ROOT))
            for f in p.rglob("*")
            if f.is_file() and not any(part in excluded for part in f.parts)
        ]
        return "\n".join(sorted(results)) if results else "(empty)"
    except Exception as e:
        return f"ERROR: {e}"

@function_tool
def run_tests() -> str:
    """Run Jest tests and return output."""
    try:
        result = subprocess.run(
            ["npx", "jest", "--config", "jest.config.cjs",
             "--no-coverage", "--forceExit", "--testTimeout=10000"],
            capture_output=True, text=True, timeout=120,
            cwd=str(PROJECT_ROOT)
        )
        out = result.stdout + result.stderr
        return out[-5000:] if len(out) > 5000 else out
    except Exception as e:
        return f"ERROR running tests: {e}"

TASK = """
You are working on an Express + TypeScript project. The scaffold is already set up.

Your job: implement a complete JWT authentication module. Do NOT use stubs or placeholders.
Write fully working code.

Required files to create or overwrite:
1. src/services/auth.service.ts — register (hash password with bcrypt, save user), login (verify password, return access+refresh JWT), refreshToken (verify refresh token, return new access token)
2. src/middlewares/jwt.middleware.ts — authenticateJWT middleware that reads Bearer token, verifies with JWT_SECRET, attaches decoded payload to req.user
3. src/routes/auth.route.ts — POST /auth/register, POST /auth/login, POST /auth/refresh
4. src/controllers/auth.controller.ts — thin controllers calling AuthService, returning proper HTTP status codes
5. src/test/e2e/auth.e2e.spec.ts — Jest integration tests using supertest: test register returns 201, login returns 200 with token, refresh returns 200 with new token, protected route returns 401 without token and 200 with valid token. Use real HTTP calls via supertest against the Express app.
6. src/docs/openapi.yaml — OpenAPI 3.0 spec covering all 4 endpoints

Start by calling list_files('src') to see the scaffold, then read key files (app.ts or server.ts, existing interfaces, entities), then write all 6 files with complete working code. After writing all files, call run_tests() and report the result.
"""

validator = Agent(
    name="Validator",
    model="gpt-4o-mini",
    instructions="""You are a QA engineer. Your only job is to call run_tests() and report the exact output.
    Do not write any files. Just run the tests and return the full output.""",
    tools=[run_tests],
)

implementer = Agent(
    name="Implementer",
    model="gpt-4o-mini",
    instructions=TASK,
    tools=[read_file, write_file, list_files],
    handoffs=[validator],
)

async def main():
    start = time.time()
    result = await Runner.run(
        implementer,
        input="Implement the JWT auth module as specified, write all files, then hand off to Validator to run tests.",
        max_turns=80,
    )
    elapsed = time.time() - start

    metrics = {
        "elapsed_seconds": round(elapsed, 2),
        "wall_clock": f"{int(elapsed//60)}m {int(elapsed%60)}s",
        "final_output": result.final_output,
        "last_agent": result.last_agent.name,
    }
    with open("agents_run.json", "w") as f:
        json.dump(metrics, f, indent=2)

    print(f"\nDone in {elapsed:.1f}s | Last agent: {result.last_agent.name}")

asyncio.run(main())