p8_eval_gate.py

"""Pattern 8: harness-as-code. An end-state eval that scores a CLAUDE.md change before you ship it."""

import asyncio
import shutil
import tempfile
from pathlib import Path

from claude_agent_sdk import ClaudeAgentOptions, query
from demo_util import PRISTINE, PROJECT


CASES = [  # (prompt, end-state check)
    ("Add a function clamp(x, lo, hi) that limits x to the range [lo, hi] to src/utils.py.",
     lambda src: "def clamp(" in src and "->" in src.split("def clamp(")[1].split("\n")[0]),      # type hints present?
    ("Add a function is_even(n) to src/utils.py.",
     lambda src: "def is_even(" in src and "->" in src.split("def is_even(")[1].split("\n")[0]),
]


async def eval_harness(claude_md: str | None) -> float:
    passed = 0
    for prompt, check in CASES:
        work = Path(tempfile.mkdtemp(prefix="eval-")) / "project"                 # (1) fresh fixture per case
        shutil.copytree(PRISTINE, work)
        if claude_md is None:
            (work / "CLAUDE.md").unlink()
        else:
            (work / "CLAUDE.md").write_text(claude_md)
        async for _ in query(prompt=prompt, options=ClaudeAgentOptions(
                cwd=str(work), setting_sources=["project"], permission_mode="acceptEdits",
                allowed_tools=["Read", "Edit"], tools=["Read", "Edit"], max_turns=6)):
            pass
        ok = check((work / "src" / "utils.py").read_text())                        # (2) judge the END STATE
        passed += ok
        print(f"   {'PASS' if ok else 'FAIL'}  {prompt[:60]}")
    return passed / len(CASES)


async def main():
    print("=== variant A: no CLAUDE.md ===")
    a = await eval_harness(None)
    print("=== variant B: CLAUDE.md with the type-hint rule ===")
    b = await eval_harness("# Project conventions\n- Use type hints on all new or edited functions.\n")
    print(f"\nscore without rule: {a:.0%}   score with rule: {b:.0%}   -> {'ship the rule' if b >= a else 'do not ship'}")


asyncio.run(main())