scripts/generate_proof_artifacts.py

#!/usr/bin/env python3

from __future__ import annotations

import json
import sys
from pathlib import Path


def main() -> int:
    root = Path(__file__).resolve().parents[1]
    sys.path.insert(0, str(root))
    from support_ops_env.inference import run_strong_in_process

    rows = []
    for tid in [
        "ds_prompt_injection_access",
        "ds_schema_drift_refund",
        "ds_poisoned_memory_case",
        "ds_lying_tool_gdpr",
    ]:
        s, _, _ = run_strong_in_process(tid)
        rows.append({"task_id": tid, "scripted_strong": round(s, 4)})

    eval_md = root / "eval_compare.md"
    body = "\n".join(
        [
            "# Eval compare (regenerated)",
            "",
            "| Task | Scripted strong (deterministic) |",
            "|------|--------------------------------:|",
        ]
        + [f"| {r['task_id']} | {r['scripted_strong']:.4f} |" for r in rows]
        + ["", f"```json\n{json.dumps(rows, indent=2)}\n```\n"],
    )
    eval_md.write_text(body, encoding="utf-8")
    print("Wrote", eval_md)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())