Update NCAR and rollout regression tests

workspace/tests/test_metrics.py

@@ -12,6 +12,7 @@ from cil.metrics import (
     candidate_diversity,
     collapse_rate,
     mean_nearest_distance_to_set,
+    normalized_causal_action_regret,
     outcome_safety_violation,
     outcome_ptr_at_k,
     proxy_positive_tangent_coverage_at_k,
@@ -36,6 +37,11 @@ def test_selector_regret_refuses_unevaluated_candidates() -> None:
         selector_regret_at_k([0.5, 0.7], candidates_evaluated=False)
 
 
+def test_normalized_causal_action_regret() -> None:
+    assert normalized_causal_action_regret(2.0, 1.25, 1.0) == pytest.approx(0.75)
+    assert normalized_causal_action_regret(2.0, 0.5, 1.0) > 1.0
+
+
 def test_proxy_positive_tangent_coverage_is_not_outcome_ptr() -> None:
     generated = [[0.0, 0.1], [2.0, 2.0]]
     positives = [[0.0, 0.0]]
@@ -150,6 +156,10 @@ def test_eval_metrics_measured_exports_success_decomposition(tmp_path: Path) ->
     assert row["hidden_chart_oracle_success_at_3"] == 1.0
     assert row["success_support_gap_at_3"] == 0.0
     assert row["success_selector_gap_at_3"] == 1.0
+    assert row["ncar_to_proposal_oracle_at_3"] == pytest.approx(1.125)
+    assert row["ncar_to_hidden_chart_oracle_at_3"] == pytest.approx(1.0833333333)
+    assert row["support_gap_fraction_to_hidden_at_3"] == pytest.approx(0.3333333333)
+    assert row["selector_gap_fraction_to_hidden_at_3"] == pytest.approx(0.75)
     assert row["proposal_oracle_utility_gain_over_base_at_3"] == pytest.approx(0.8)
     assert row["base_safety_label_known"] == 1.0
     assert row["base_unsafe_known"] == 0.0
@@ -223,6 +233,49 @@ def test_eval_metrics_measured_exports_safety_coverage_without_false_zero(
     assert "proposal_oracle_unsafe_known_at_2" not in row
 
 
+def test_eval_metrics_omits_unstable_ncar_when_oracle_matches_base(tmp_path: Path) -> None:
+    input_path = tmp_path / "rows.json"
+    input_path.write_text(
+        json.dumps(
+            {
+                "rows": [
+                    {
+                        "chart_id": "c0",
+                        "task_id": "pick",
+                        "seed": 0,
+                        "candidates_evaluated": True,
+                        "base_utility": 0.5,
+                        "generated_utilities": [0.4, 0.5],
+                        "selected_index": 0,
+                    }
+                ]
+            }
+        )
+    )
+    out_dir = tmp_path / "measured_metrics_unstable_ncar"
+
+    assert (
+        eval_metrics.main(
+            [
+                "--input",
+                str(input_path),
+                "--out-dir",
+                str(out_dir),
+                "--mode",
+                "measured",
+                "--k",
+                "2",
+                "--bootstrap-samples",
+                "20",
+            ]
+        )
+        == 0
+    )
+
+    row = json.loads((out_dir / "metrics.json").read_text())["rows"][0]
+    assert "ncar_to_proposal_oracle_at_2" not in row
+
+
 def test_eval_metrics_proxy_exports_json_and_latex(tmp_path: Path) -> None:
     input_path = tmp_path / "rows.json"
     input_path.write_text(