scripts/run_baseline_eval.py

"""Run rule-based policies × scenarios eval.

Usage:
    python scripts/run_baseline_eval.py --out eval_outputs/baseline
"""
from __future__ import annotations

import argparse
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from eval.harness import run_eval_matrix
from eval.policies import build_policy


DEFAULT_POLICIES = [
    "random", "blanket_approve_pr", "blanket_reject_pr",
    "keyword_security_triager", "paranoid_inspector", "memory_aware",
]
DEFAULT_SCENARIOS = [
    "E0_normal_intake", "E1_typosquat_wave", "E2_social_eng_buildup",
    "E3_compromised_maintainer", "E4_multi_vector",
]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default="eval_outputs/baseline")
    ap.add_argument("--policies", nargs="*", default=DEFAULT_POLICIES)
    ap.add_argument("--scenarios", nargs="*", default=DEFAULT_SCENARIOS)
    ap.add_argument("--seeds", nargs="*", type=int, default=[0, 1, 2])
    args = ap.parse_args()

    policies = {n: build_policy(n) for n in args.policies}
    out_dir = Path(args.out)
    records = run_eval_matrix(policies, args.scenarios, args.seeds, out_dir, verbose=True)
    print(f"\n[done] {len(records)} rollouts -> {out_dir}")
    print(f"  summary.md: {out_dir / 'summary.md'}")


if __name__ == "__main__":
    main()