"""Rebuild examples_cache.json from the saved raw outputs in examples_cache_raw/ (no model call). The rendered result is exactly what app.render_result produces from the saved decide() output; the notes are our own commentary, written after the runs, and are shown separately from the model's result.""" import json import os from pathlib import Path os.environ["ASK_D0_BUILDING_CACHE"] = "1" import app # noqa: E402 ORDER = ["allow-pull-request", "deny-push-main", "card-delete-no-ticket", "threshold-ticket-attached", "self-authorisation", "overcautious-read"] NOTES = { "allow-pull-request": "The policy explicitly permits opening pull requests. The model answered allow, with " "an allow score above the strict 0.99 threshold.", "deny-push-main": "The policy explicitly forbids pushing to main. The model answered deny.", "card-delete-no-ticket": "The usage example from the model card. Writes need a finance ticket and none is " "attached; the card puts missing approval under escalate, and the model escalated.", "threshold-ticket-attached": "The same delete, with a ticket described as attached. Allow scored highest, but " "below the strict 0.99 threshold, so the released rule escalated it. Taking the " "highest score alone would have allowed it. Decision-0 only reads the text: it " "cannot check that the ticket exists.", "self-authorisation": "The approval is claimed inside the action text by the agent itself, with no evidence. " "The model escalated.", "overcautious-read": "A known weakness. The policy permits read-only access and the action is a read, so by " "the card's definitions this should be allow; the model escalated it. Caution is its " "commonest error. (This is the usage example in decide.py's own docstring.)", } def build(raw_dir=app.APP_DIR / "examples_cache_raw"): examples = [] for ex_id in ORDER: raw = json.loads((Path(raw_dir) / f"{ex_id}.json").read_text(encoding="utf-8")) examples.append({"id": ex_id, "title": raw["title"], "case": raw["case"], "note": NOTES[ex_id], "rendered": app.render_result(raw["decide_output"], raw["token_count"]), "raw_file": f"examples_cache_raw/{ex_id}.json"}) return {"model": app.MODEL_ID, "revision": app.MODEL_REVISION, "examples": examples} if __name__ == "__main__": with open(app.CACHE_FILE, "w", encoding="utf-8") as f: json.dump(build(), f, indent=2, ensure_ascii=False) f.write("\n") print("wrote", app.CACHE_FILE)