Update rollout metrics scripts and HF sync helpers

workspace/scripts/build_ctt_rollout_comparison.py

@@ -17,6 +17,7 @@ if str(PROJECT_ROOT) not in sys.path:
 
 from cil.metrics import (  # noqa: E402
     any_unsafe,
+    normalized_causal_action_regret,
     outcome_safety_violation,
     safety_label_coverage,
     unsafe_rate,
@@ -161,6 +162,10 @@ def _report(combined: dict[str, Any], metrics: dict[str, Any]) -> str:
             f"| selected_utility_mean | {_fmt(success.get('selected_utility_mean'))} |",
             f"| proposal_oracle_utility_mean | {_fmt(success.get('proposal_oracle_utility_mean'))} |",
             f"| hidden_chart_oracle_utility_mean | {_fmt(success.get('hidden_chart_oracle_utility_mean'))} |",
+            f"| ncar_to_proposal_oracle | {_fmt(success.get('ncar_to_proposal_oracle'))} |",
+            f"| ncar_to_hidden_chart_oracle | {_fmt(success.get('ncar_to_hidden_chart_oracle'))} |",
+            f"| utility_support_gap_fraction_to_hidden | {_fmt(success.get('utility_support_gap_fraction_to_hidden'))} |",
+            f"| utility_selector_gap_fraction_to_hidden | {_fmt(success.get('utility_selector_gap_fraction_to_hidden'))} |",
             f"| generated_safety_label_coverage | {_fmt(success.get('generated_safety_label_coverage'))} |",
             f"| generated_unsafe_rate_known | {_fmt(success.get('generated_unsafe_rate_known'))} |",
             f"| any_generated_unsafe_known | {_fmt(success.get('any_generated_unsafe_known'))} |",
@@ -212,6 +217,10 @@ def _success_summary(rows: list[dict[str, Any]], *, k: int) -> dict[str, Any]:
     oracle_utility = []
     hidden_oracle_utility = []
     hidden_oracle_success = []
+    ncar_to_proposal_oracle = []
+    ncar_to_hidden_chart_oracle = []
+    utility_support_gap_fraction = []
+    utility_selector_gap_fraction = []
     success_support_gap = []
     success_selector_gap = []
     selected_success_gain = []
@@ -273,14 +282,56 @@ def _success_summary(rows: list[dict[str, Any]], *, k: int) -> dict[str, Any]:
             proposal_oracle_success_value = float(any(generated_success))
             oracle_success.append(proposal_oracle_success_value)
         if "base_utility" in row:
-            base_utility.append(float(row["base_utility"]))
+            base_utility_value = float(row["base_utility"])
+            base_utility.append(base_utility_value)
+        else:
+            base_utility_value = None
         if selected_index < len(generated_utilities):
-            selected_utility.append(generated_utilities[selected_index])
+            selected_utility_value = generated_utilities[selected_index]
+            selected_utility.append(selected_utility_value)
+        else:
+            selected_utility_value = None
         if generated_utilities:
-            oracle_utility.append(max(generated_utilities))
+            proposal_oracle_utility_value = max(generated_utilities)
+            oracle_utility.append(proposal_oracle_utility_value)
+        else:
+            proposal_oracle_utility_value = None
+        if (
+            base_utility_value is not None
+            and selected_utility_value is not None
+            and proposal_oracle_utility_value is not None
+        ):
+            _append_stable_ncar(
+                ncar_to_proposal_oracle,
+                proposal_oracle_utility_value,
+                selected_utility_value,
+                base_utility_value,
+            )
         hidden = [float(value) for value in row.get("hidden_chart_utilities", [])]
         if hidden:
-            hidden_oracle_utility.append(max(hidden))
+            hidden_oracle_utility_value = max(hidden)
+            hidden_oracle_utility.append(hidden_oracle_utility_value)
+            if (
+                base_utility_value is not None
+                and selected_utility_value is not None
+                and proposal_oracle_utility_value is not None
+            ):
+                _append_stable_ncar(
+                    ncar_to_hidden_chart_oracle,
+                    hidden_oracle_utility_value,
+                    selected_utility_value,
+                    base_utility_value,
+                )
+                hidden_gap = abs(hidden_oracle_utility_value - base_utility_value)
+                if hidden_gap > 0.0:
+                    utility_support_gap_fraction.append(
+                        max(0.0, hidden_oracle_utility_value - proposal_oracle_utility_value)
+                        / hidden_gap
+                    )
+                    utility_selector_gap_fraction.append(
+                        max(0.0, proposal_oracle_utility_value - selected_utility_value)
+                        / hidden_gap
+                    )
             hidden_success_value = float(any(value >= 1.0 for value in hidden))
             hidden_oracle_success.append(hidden_success_value)
             if proposal_oracle_success_value is not None:
@@ -315,6 +366,10 @@ def _success_summary(rows: list[dict[str, Any]], *, k: int) -> dict[str, Any]:
         "selected_utility_mean": _mean(selected_utility),
         "proposal_oracle_utility_mean": _mean(oracle_utility),
         "hidden_chart_oracle_utility_mean": _mean(hidden_oracle_utility),
+        "ncar_to_proposal_oracle": _mean(ncar_to_proposal_oracle),
+        "ncar_to_hidden_chart_oracle": _mean(ncar_to_hidden_chart_oracle),
+        "utility_support_gap_fraction_to_hidden": _mean(utility_support_gap_fraction),
+        "utility_selector_gap_fraction_to_hidden": _mean(utility_selector_gap_fraction),
         "generated_safety_label_coverage": _mean(generated_safety_coverage),
         "generated_unsafe_rate_known": _mean(generated_unsafe),
         "any_generated_unsafe_known": _mean(any_generated_unsafe),
@@ -339,6 +394,25 @@ def _mean(values: list[float]) -> float | None:
     return sum(clean) / len(clean) if clean else None
 
 
+def _append_stable_ncar(
+    output: list[float],
+    oracle_utility: float,
+    selected_utility: float,
+    base_utility: float,
+    *,
+    min_denominator: float = 1.0e-3,
+) -> None:
+    if abs(float(oracle_utility) - float(base_utility)) <= min_denominator:
+        return
+    output.append(
+        normalized_causal_action_regret(
+            oracle_utility,
+            selected_utility,
+            base_utility,
+        )
+    )
+
+
 def _fmt(value: Any) -> str:
     if not isinstance(value, (int, float)) or not math.isfinite(float(value)):
         return "n/a"

workspace/scripts/eval_metrics.py

@@ -23,6 +23,7 @@ from cil.metrics import (  # noqa: E402
     mean_nearest_distance_to_set,
     measured_support_gap,
     negative_near_at_threshold,
+    normalized_causal_action_regret,
     outcome_ptr_at_k,
     pairwise_causal_dominance_ece,
     positives_closer_than_negatives,
@@ -156,6 +157,13 @@ def _measured_row(row: dict[str, Any], *, k: int, epsilon: float) -> dict[str, A
         candidates_evaluated=True,
     )
     output[f"branch_car_at_{k}"] = branch_car(max(prefix), selected_utility)
+    ncar_to_proposal = _stable_ncar(
+        proposal_oracle_utility,
+        selected_utility,
+        base_utility,
+    )
+    if ncar_to_proposal is not None:
+        output[f"ncar_to_proposal_oracle_at_{k}"] = ncar_to_proposal
     output["base_utility"] = base_utility
     output[f"selected_utility_at_{k}"] = selected_utility
     output[f"proposal_oracle_utility_at_{k}"] = proposal_oracle_utility
@@ -230,6 +238,21 @@ def _measured_row(row: dict[str, Any], *, k: int, epsilon: float) -> dict[str, A
             hidden_oracle_utility,
             selected_utility,
         )
+        ncar_to_hidden = _stable_ncar(
+            hidden_oracle_utility,
+            selected_utility,
+            base_utility,
+        )
+        if ncar_to_hidden is not None:
+            output[f"ncar_to_hidden_chart_oracle_at_{k}"] = ncar_to_hidden
+        hidden_gap = abs(hidden_oracle_utility - base_utility)
+        if hidden_gap > 0.0:
+            output[f"support_gap_fraction_to_hidden_at_{k}"] = (
+                output[f"support_gap_at_{k}"] / hidden_gap
+            )
+            output[f"selector_gap_fraction_to_hidden_at_{k}"] = (
+                output[f"selector_regret_at_{k}"] / hidden_gap
+            )
         if candidate_success:
             hidden_oracle_success = float(any(value >= 1.0 for value in hidden))
             output[f"hidden_chart_oracle_success_at_{k}"] = hidden_oracle_success
@@ -332,6 +355,24 @@ def _optional_bool(value: Any) -> bool | None:
     return bool(value)
 
 
+def _stable_ncar(
+    oracle_utility: float,
+    selected_utility: float,
+    base_utility: float,
+    *,
+    min_denominator: float = 1.0e-3,
+) -> float | None:
+    """Return NCAR only when the base-to-oracle gap is numerically meaningful."""
+
+    if abs(float(oracle_utility) - float(base_utility)) <= min_denominator:
+        return None
+    return normalized_causal_action_regret(
+        oracle_utility,
+        selected_utility,
+        base_utility,
+    )
+
+
 def _matrix(row: dict[str, Any], key: str, *, required: bool = True) -> list[list[float]]:
     values = row.get(key)
     if values is None: