eval/build_testset.py

"""Generate `eval/testset.json` from the campaigns' private `_expected` ground truth.

Each campaign file carries a private `_expected` block (recommendation + the rule IDs a
correct adjudication should rest on). That block is stripped before the agent ever sees the
campaign (`src/campaigns.load_campaign`); only the eval harness is allowed to read it via
`load_expected`. This script collects those blocks into a single committed test set so the
eval and CI have a stable, reviewable ground-truth file.

    python -m eval.build_testset            # writes eval/testset.json
    python -m eval.build_testset --check    # verify the committed file is up to date (CI)
"""
from __future__ import annotations

import argparse
import json
from pathlib import Path

from src.campaigns import list_campaign_paths, load_expected

TESTSET_PATH = Path("eval/testset.json")


def build() -> list[dict]:
    """Collect every campaign's `_expected` into a list of test cases."""
    cases: list[dict] = []
    for path in list_campaign_paths():
        expected = load_expected(path)
        if not expected:
            continue  # a campaign without ground truth is not part of the eval set
        cases.append(
            {
                "id": path.stem,
                "path": path.as_posix(),
                "expected": {
                    "recommendation": expected["recommendation"],
                    "primary_rules": expected.get("primary_rules", []),
                },
            }
        )
    return cases


def _dump(cases: list[dict]) -> str:
    return json.dumps(cases, indent=2, ensure_ascii=False) + "\n"


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "--check",
        action="store_true",
        help="Exit non-zero if eval/testset.json is missing or out of date (do not rewrite).",
    )
    args = parser.parse_args()

    cases = build()
    rendered = _dump(cases)

    if args.check:
        current = TESTSET_PATH.read_text(encoding="utf-8") if TESTSET_PATH.exists() else ""
        if current != rendered:
            raise SystemExit(
                f"{TESTSET_PATH} is out of date — run `python -m eval.build_testset` and commit."
            )
        print(f"{TESTSET_PATH} is up to date ({len(cases)} cases).")
        return

    TESTSET_PATH.write_text(rendered, encoding="utf-8")
    print(f"Wrote {TESTSET_PATH} ({len(cases)} cases).")


if __name__ == "__main__":
    main()