| |
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import math |
| import re |
| import shutil |
| import subprocess |
| import sys |
| from pathlib import Path |
| from typing import Any |
|
|
| PROJECT_ROOT = Path(__file__).resolve().parents[1] |
| if str(PROJECT_ROOT) not in sys.path: |
| sys.path.insert(0, str(PROJECT_ROOT)) |
|
|
| from cil.metrics import ( |
| any_unsafe, |
| normalized_causal_action_regret, |
| outcome_safety_violation, |
| safety_label_coverage, |
| unsafe_rate, |
| ) |
| from scripts.eval_metrics import main as eval_metrics_main |
|
|
|
|
| def main(argv: list[str] | None = None) -> int: |
| parser = argparse.ArgumentParser(description="Aggregate measured CTT rollout runs.") |
| parser.add_argument( |
| "--run-glob", |
| default="runs/ctt_residual_rollout_val69_seed*", |
| help="Glob for run directories containing measured_candidates.json.", |
| ) |
| parser.add_argument("--out-dir", type=Path, default=Path("runs/ctt_val_rollout_comparison")) |
| parser.add_argument("--k", type=int, default=8) |
| parser.add_argument("--bootstrap-samples", type=int, default=1000) |
| parser.add_argument( |
| "--no-markdown-report", |
| action="store_true", |
| help="Do not write report.md; persistent prose lives in README.md.", |
| ) |
| args = parser.parse_args(argv) |
|
|
| run_dirs = [ |
| path |
| for path in sorted(Path().glob(args.run_glob)) |
| if (path / "measured_candidates.json").exists() |
| ] |
| if not run_dirs: |
| raise SystemExit(f"no measured rollout runs found for glob {args.run_glob!r}") |
|
|
| combined_rows: list[dict[str, Any]] = [] |
| payloads = [] |
| for run_dir in run_dirs: |
| payload = json.loads((run_dir / "measured_candidates.json").read_text()) |
| payloads.append(payload) |
| train_seed = _seed_from_path(run_dir) |
| for row in payload.get("rows", []): |
| item = dict(row) |
| item["train_seed"] = train_seed |
| item["seed"] = f"{row.get('seed', 'unknown')}/train{train_seed}" |
| combined_rows.append(item) |
|
|
| out_dir = args.out_dir |
| out_dir.mkdir(parents=True, exist_ok=True) |
| combined = { |
| "report_type": "ctt_measured_rollout_comparison", |
| "schema_version": 1, |
| "k": args.k, |
| "run_dirs": [str(path) for path in run_dirs], |
| "train_seeds": [_seed_from_path(path) for path in run_dirs], |
| "num_rows": len(combined_rows), |
| "data_hash": _first(payloads, "target_content_hash"), |
| "split_hash": _first(payloads, "target_split_hash"), |
| "source_content_hash": _first(payloads, "source_content_hash"), |
| "target_content_hash": _first(payloads, "target_content_hash"), |
| "target_split_hash": _first(payloads, "target_split_hash"), |
| "rows": combined_rows, |
| } |
| combined_path = out_dir / "combined_measured_candidates.json" |
| combined_path.write_text(json.dumps(combined, indent=2, sort_keys=True) + "\n") |
| metric_args = [ |
| "--input", |
| str(combined_path), |
| "--out-dir", |
| str(out_dir / "measured_metrics"), |
| "--mode", |
| "measured", |
| "--k", |
| str(args.k), |
| "--bootstrap-samples", |
| str(args.bootstrap_samples), |
| ] |
| if args.no_markdown_report: |
| metric_args.append("--no-markdown-report") |
| eval_metrics_main(metric_args) |
| metrics = json.loads((out_dir / "measured_metrics" / "metrics.json").read_text()) |
| (out_dir / "metrics.json").write_text(json.dumps(_summary_payload(combined, metrics), indent=2, sort_keys=True) + "\n") |
| (out_dir / "table.tex").write_text((out_dir / "measured_metrics" / "table.tex").read_text()) |
| report_path = out_dir / "report.md" |
| if args.no_markdown_report: |
| report_path.unlink(missing_ok=True) |
| else: |
| report_path.write_text(_report(combined, metrics) + "\n") |
| for filename in ("metrics_by_task.json", "metrics_by_seed.json"): |
| shutil.copyfile(out_dir / "measured_metrics" / filename, out_dir / filename) |
| (out_dir / "command.txt").write_text( |
| "python scripts/build_ctt_rollout_comparison.py " + " ".join(sys.argv[1:]) + "\n" |
| ) |
| (out_dir / "git_hash.txt").write_text(_run(["git", "rev-parse", "HEAD"]) + "\n") |
| (out_dir / "data_hash.txt").write_text(str(combined.get("target_content_hash") or "") + "\n") |
| (out_dir / "split_hash.txt").write_text(str(combined.get("target_split_hash") or "") + "\n") |
| (out_dir / "source_data_hash.txt").write_text(str(combined.get("source_content_hash") or "") + "\n") |
| (out_dir / "train.log").write_text("comparison artifact; source runs trained/evaluated separately\n") |
| (out_dir / "eval.log").write_text( |
| "\n".join(f"{run_dir}: measured_candidates.json" for run_dir in combined["run_dirs"]) + "\n" |
| ) |
| print(json.dumps({"out_dir": str(out_dir), "runs": len(run_dirs), "rows": len(combined_rows)}, indent=2)) |
| return 0 |
|
|
|
|
| def _summary_payload(combined: dict[str, Any], metrics: dict[str, Any]) -> dict[str, Any]: |
| return { |
| "report_type": "ctt_measured_rollout_comparison", |
| "k": combined["k"], |
| "run_dirs": combined["run_dirs"], |
| "train_seeds": combined["train_seeds"], |
| "num_rows": combined["num_rows"], |
| "data_hash": combined.get("target_content_hash"), |
| "split_hash": combined.get("target_split_hash"), |
| "summary": metrics.get("summary", {}), |
| "success_summary": _success_summary(combined.get("rows", []), k=int(combined["k"])), |
| "source_content_hash": combined.get("source_content_hash"), |
| "target_content_hash": combined.get("target_content_hash"), |
| "target_split_hash": combined.get("target_split_hash"), |
| } |
|
|
|
|
| def _report(combined: dict[str, Any], metrics: dict[str, Any]) -> str: |
| summary = metrics.get("summary", {}) |
| split = _split_name(combined) |
| lines = [ |
| f"# CTT {split.title()} Measured Rollout Comparison", |
| "", |
| f"Runs: `{len(combined['run_dirs'])}`", |
| f"Rows: `{combined['num_rows']}`", |
| f"K: `{combined['k']}`", |
| "", |
| "| Metric | N | Micro mean | 95% CI |", |
| "| --- | ---: | ---: | ---: |", |
| ] |
| for name, payload in sorted(summary.items()): |
| micro = payload.get("micro", {}) |
| lines.append( |
| f"| {name} | {micro.get('n', 0)} | {_fmt(micro.get('mean'))} | " |
| f"[{_fmt(micro.get('low'))}, {_fmt(micro.get('high'))}] |" |
| ) |
| lines.append("") |
| success = _success_summary(combined.get("rows", []), k=int(combined["k"])) |
| lines.extend( |
| [ |
| "| Success/Utility | Mean |", |
| "| --- | ---: |", |
| f"| base_success_rate | {_fmt(success.get('base_success_rate'))} |", |
| f"| selected_success_rate | {_fmt(success.get('selected_success_rate'))} |", |
| f"| proposal_oracle_success_rate | {_fmt(success.get('proposal_oracle_success_rate'))} |", |
| f"| hidden_chart_oracle_success_rate | {_fmt(success.get('hidden_chart_oracle_success_rate'))} |", |
| f"| selected_success_gain_over_base | {_fmt(success.get('selected_success_gain_over_base'))} |", |
| f"| proposal_oracle_success_gain_over_base | {_fmt(success.get('proposal_oracle_success_gain_over_base'))} |", |
| f"| success_support_gap | {_fmt(success.get('success_support_gap'))} |", |
| f"| success_selector_gap | {_fmt(success.get('success_selector_gap'))} |", |
| f"| base_utility_mean | {_fmt(success.get('base_utility_mean'))} |", |
| f"| selected_utility_mean | {_fmt(success.get('selected_utility_mean'))} |", |
| f"| proposal_oracle_utility_mean | {_fmt(success.get('proposal_oracle_utility_mean'))} |", |
| f"| hidden_chart_oracle_utility_mean | {_fmt(success.get('hidden_chart_oracle_utility_mean'))} |", |
| f"| ncar_to_proposal_oracle | {_fmt(success.get('ncar_to_proposal_oracle'))} |", |
| f"| ncar_to_hidden_chart_oracle | {_fmt(success.get('ncar_to_hidden_chart_oracle'))} |", |
| f"| utility_support_gap_fraction_to_hidden | {_fmt(success.get('utility_support_gap_fraction_to_hidden'))} |", |
| f"| utility_selector_gap_fraction_to_hidden | {_fmt(success.get('utility_selector_gap_fraction_to_hidden'))} |", |
| f"| generated_safety_label_coverage | {_fmt(success.get('generated_safety_label_coverage'))} |", |
| f"| generated_unsafe_rate_known | {_fmt(success.get('generated_unsafe_rate_known'))} |", |
| f"| any_generated_unsafe_known | {_fmt(success.get('any_generated_unsafe_known'))} |", |
| f"| selected_safety_label_known_rate | {_fmt(success.get('selected_safety_label_known_rate'))} |", |
| f"| selected_unsafe_rate_known | {_fmt(success.get('selected_unsafe_rate_known'))} |", |
| f"| proposal_oracle_safety_label_known_rate | {_fmt(success.get('proposal_oracle_safety_label_known_rate'))} |", |
| f"| proposal_oracle_unsafe_rate_known | {_fmt(success.get('proposal_oracle_unsafe_rate_known'))} |", |
| f"| base_safety_label_known_rate | {_fmt(success.get('base_safety_label_known_rate'))} |", |
| f"| base_unsafe_rate_known | {_fmt(success.get('base_unsafe_rate_known'))} |", |
| "", |
| ] |
| ) |
| lines.append("These are measured generated-candidate rollouts, not PPTC proxies.") |
| lines.append("") |
| lines.append("Run dirs:") |
| for run_dir in combined["run_dirs"]: |
| lines.append(f"- `{run_dir}`") |
| return "\n".join(lines) |
|
|
|
|
| def _seed_from_path(path: Path) -> str: |
| match = re.search(r"seed(\d+)", path.name) |
| return match.group(1) if match else path.name |
|
|
|
|
| def _split_name(combined: dict[str, Any]) -> str: |
| text = " ".join(str(item) for item in combined.get("run_dirs", [])) |
| if "test" in text: |
| return "test" |
| if "val" in text or "validation" in text: |
| return "validation" |
| return "measured" |
|
|
|
|
| def _first(payloads: list[dict[str, Any]], key: str) -> Any: |
| for payload in payloads: |
| value = payload.get(key) |
| if value: |
| return value |
| return None |
|
|
|
|
| def _success_summary(rows: list[dict[str, Any]], *, k: int) -> dict[str, Any]: |
| base_success = [] |
| selected_success = [] |
| oracle_success = [] |
| base_utility = [] |
| selected_utility = [] |
| oracle_utility = [] |
| hidden_oracle_utility = [] |
| hidden_oracle_success = [] |
| ncar_to_proposal_oracle = [] |
| ncar_to_hidden_chart_oracle = [] |
| utility_support_gap_fraction = [] |
| utility_selector_gap_fraction = [] |
| success_support_gap = [] |
| success_selector_gap = [] |
| selected_success_gain = [] |
| proposal_oracle_success_gain = [] |
| restore_errors = [] |
| generated_safety_coverage = [] |
| generated_unsafe = [] |
| any_generated_unsafe = [] |
| selected_safety_known = [] |
| selected_unsafe = [] |
| proposal_oracle_safety_known = [] |
| proposal_oracle_unsafe = [] |
| base_safety_known = [] |
| base_unsafe = [] |
| for row in rows: |
| generated_utilities = [float(value) for value in row.get("generated_utilities", [])[:k]] |
| generated_success = [bool(value) for value in row.get("candidate_success", [])[:k]] |
| candidate_outcomes = _outcome_list(row.get("candidate_outcomes", []))[:k] |
| selected_index = int(row.get("selected_index", 0)) |
| selected_success_value: float | None = None |
| proposal_oracle_success_value: float | None = None |
| base_success_value: float | None = None |
| base_outcome = row.get("base_outcome") |
| if isinstance(base_outcome, dict): |
| safety = outcome_safety_violation(base_outcome) |
| base_safety_known.append(float(safety is not None)) |
| if safety is not None: |
| base_unsafe.append(float(safety)) |
| if candidate_outcomes: |
| generated_safety_coverage.append(safety_label_coverage(candidate_outcomes, k=k)) |
| unsafe = unsafe_rate(candidate_outcomes, k=k) |
| if unsafe is not None: |
| generated_unsafe.append(unsafe) |
| any_unsafe_value = any_unsafe(candidate_outcomes, k=k) |
| if any_unsafe_value is not None: |
| any_generated_unsafe.append(any_unsafe_value) |
| if selected_index < len(candidate_outcomes): |
| safety = outcome_safety_violation(candidate_outcomes[selected_index]) |
| selected_safety_known.append(float(safety is not None)) |
| if safety is not None: |
| selected_unsafe.append(float(safety)) |
| if generated_utilities: |
| oracle_index = max( |
| range(len(generated_utilities)), |
| key=lambda index: generated_utilities[index], |
| ) |
| if oracle_index < len(candidate_outcomes): |
| safety = outcome_safety_violation(candidate_outcomes[oracle_index]) |
| proposal_oracle_safety_known.append(float(safety is not None)) |
| if safety is not None: |
| proposal_oracle_unsafe.append(float(safety)) |
| if "base_success" in row: |
| base_success_value = float(bool(row["base_success"])) |
| base_success.append(base_success_value) |
| if selected_index < len(generated_success): |
| selected_success_value = float(generated_success[selected_index]) |
| selected_success.append(selected_success_value) |
| if generated_success: |
| proposal_oracle_success_value = float(any(generated_success)) |
| oracle_success.append(proposal_oracle_success_value) |
| if "base_utility" in row: |
| base_utility_value = float(row["base_utility"]) |
| base_utility.append(base_utility_value) |
| else: |
| base_utility_value = None |
| if selected_index < len(generated_utilities): |
| selected_utility_value = generated_utilities[selected_index] |
| selected_utility.append(selected_utility_value) |
| else: |
| selected_utility_value = None |
| if generated_utilities: |
| proposal_oracle_utility_value = max(generated_utilities) |
| oracle_utility.append(proposal_oracle_utility_value) |
| else: |
| proposal_oracle_utility_value = None |
| if ( |
| base_utility_value is not None |
| and selected_utility_value is not None |
| and proposal_oracle_utility_value is not None |
| ): |
| _append_stable_ncar( |
| ncar_to_proposal_oracle, |
| proposal_oracle_utility_value, |
| selected_utility_value, |
| base_utility_value, |
| ) |
| hidden = [float(value) for value in row.get("hidden_chart_utilities", [])] |
| if hidden: |
| hidden_oracle_utility_value = max(hidden) |
| hidden_oracle_utility.append(hidden_oracle_utility_value) |
| if ( |
| base_utility_value is not None |
| and selected_utility_value is not None |
| and proposal_oracle_utility_value is not None |
| ): |
| _append_stable_ncar( |
| ncar_to_hidden_chart_oracle, |
| hidden_oracle_utility_value, |
| selected_utility_value, |
| base_utility_value, |
| ) |
| hidden_gap = abs(hidden_oracle_utility_value - base_utility_value) |
| if hidden_gap > 0.0: |
| utility_support_gap_fraction.append( |
| max(0.0, hidden_oracle_utility_value - proposal_oracle_utility_value) |
| / hidden_gap |
| ) |
| utility_selector_gap_fraction.append( |
| max(0.0, proposal_oracle_utility_value - selected_utility_value) |
| / hidden_gap |
| ) |
| hidden_success_value = float(any(value >= 1.0 for value in hidden)) |
| hidden_oracle_success.append(hidden_success_value) |
| if proposal_oracle_success_value is not None: |
| success_support_gap.append( |
| max(0.0, hidden_success_value - proposal_oracle_success_value) |
| ) |
| if ( |
| proposal_oracle_success_value is not None |
| and selected_success_value is not None |
| ): |
| success_selector_gap.append( |
| max(0.0, proposal_oracle_success_value - selected_success_value) |
| ) |
| if base_success_value is not None and selected_success_value is not None: |
| selected_success_gain.append(selected_success_value - base_success_value) |
| if base_success_value is not None and proposal_oracle_success_value is not None: |
| proposal_oracle_success_gain.append( |
| proposal_oracle_success_value - base_success_value |
| ) |
| if "restore_error" in row: |
| restore_errors.append(float(row["restore_error"])) |
| return { |
| "base_success_rate": _mean(base_success), |
| "selected_success_rate": _mean(selected_success), |
| "proposal_oracle_success_rate": _mean(oracle_success), |
| "hidden_chart_oracle_success_rate": _mean(hidden_oracle_success), |
| "selected_success_gain_over_base": _mean(selected_success_gain), |
| "proposal_oracle_success_gain_over_base": _mean(proposal_oracle_success_gain), |
| "success_support_gap": _mean(success_support_gap), |
| "success_selector_gap": _mean(success_selector_gap), |
| "base_utility_mean": _mean(base_utility), |
| "selected_utility_mean": _mean(selected_utility), |
| "proposal_oracle_utility_mean": _mean(oracle_utility), |
| "hidden_chart_oracle_utility_mean": _mean(hidden_oracle_utility), |
| "ncar_to_proposal_oracle": _mean(ncar_to_proposal_oracle), |
| "ncar_to_hidden_chart_oracle": _mean(ncar_to_hidden_chart_oracle), |
| "utility_support_gap_fraction_to_hidden": _mean(utility_support_gap_fraction), |
| "utility_selector_gap_fraction_to_hidden": _mean(utility_selector_gap_fraction), |
| "generated_safety_label_coverage": _mean(generated_safety_coverage), |
| "generated_unsafe_rate_known": _mean(generated_unsafe), |
| "any_generated_unsafe_known": _mean(any_generated_unsafe), |
| "selected_safety_label_known_rate": _mean(selected_safety_known), |
| "selected_unsafe_rate_known": _mean(selected_unsafe), |
| "proposal_oracle_safety_label_known_rate": _mean(proposal_oracle_safety_known), |
| "proposal_oracle_unsafe_rate_known": _mean(proposal_oracle_unsafe), |
| "base_safety_label_known_rate": _mean(base_safety_known), |
| "base_unsafe_rate_known": _mean(base_unsafe), |
| "max_restore_error": max(restore_errors) if restore_errors else None, |
| } |
|
|
|
|
| def _outcome_list(value: Any) -> list[dict[str, Any]]: |
| if not isinstance(value, list): |
| return [] |
| return [item for item in value if isinstance(item, dict)] |
|
|
|
|
| def _mean(values: list[float]) -> float | None: |
| clean = [float(value) for value in values if math.isfinite(float(value))] |
| return sum(clean) / len(clean) if clean else None |
|
|
|
|
| def _append_stable_ncar( |
| output: list[float], |
| oracle_utility: float, |
| selected_utility: float, |
| base_utility: float, |
| *, |
| min_denominator: float = 1.0e-3, |
| ) -> None: |
| if abs(float(oracle_utility) - float(base_utility)) <= min_denominator: |
| return |
| output.append( |
| normalized_causal_action_regret( |
| oracle_utility, |
| selected_utility, |
| base_utility, |
| ) |
| ) |
|
|
|
|
| def _fmt(value: Any) -> str: |
| if not isinstance(value, (int, float)) or not math.isfinite(float(value)): |
| return "n/a" |
| return f"{float(value):.4f}" |
|
|
|
|
| def _run(command: list[str]) -> str: |
| try: |
| return subprocess.check_output(command, cwd=PROJECT_ROOT, text=True).strip() |
| except (subprocess.CalledProcessError, FileNotFoundError): |
| return "" |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
