Report pairwise calibration ECE for nonlinear selector

workspace/scripts/eval_nonlinear_dominance_selector.py

@@ -36,10 +36,13 @@ from scripts.eval_learned_dominance_selector import (  # noqa: E402
     _evaluate_predictions,
     _feature_names,
     _group_means,
+    _pairwise_calibration_global,
+    _pairwise_calibration_summary,
     _resolve_index_path,
     _selector_chart_map,
     _simple_summary,
     _source_evidence_map,
+    _summary_with_pairwise,
     _uses_chart_compat,
     _uses_source_evidence,
 )
@@ -197,11 +200,32 @@ def main(argv: list[str] | None = None) -> int:
         seed=args.seed,
     )
     eval_predictions = _predict(best["model"], eval_dataset, model_type=best["model_type"])
-    eval_cases = _evaluate_predictions(eval_dataset, eval_predictions, tau=best["tau"])
     fit_predictions = _predict(best["model"], fit_dataset, model_type=best["model_type"])
     select_predictions = _predict(best["model"], select_dataset, model_type=best["model_type"])
-    fit_cases = _evaluate_predictions(fit_dataset, fit_predictions, tau=best["tau"])
-    select_cases = _evaluate_predictions(select_dataset, select_predictions, tau=best["tau"])
+    eval_pairwise = _pairwise_calibration_summary(eval_dataset, eval_predictions)
+    fit_pairwise = _pairwise_calibration_summary(fit_dataset, fit_predictions)
+    select_pairwise = _pairwise_calibration_summary(select_dataset, select_predictions)
+    eval_cases = _evaluate_predictions(
+        eval_dataset,
+        eval_predictions,
+        tau=best["tau"],
+        include_pairwise_calibration=True,
+        pairwise_calibration=eval_pairwise,
+    )
+    fit_cases = _evaluate_predictions(
+        fit_dataset,
+        fit_predictions,
+        tau=best["tau"],
+        include_pairwise_calibration=True,
+        pairwise_calibration=fit_pairwise,
+    )
+    select_cases = _evaluate_predictions(
+        select_dataset,
+        select_predictions,
+        tau=best["tau"],
+        include_pairwise_calibration=True,
+        pairwise_calibration=select_pairwise,
+    )
 
     metric_names = sorted(
         {
@@ -264,9 +288,14 @@ def main(argv: list[str] | None = None) -> int:
         "num_selection_candidates": len(select_dataset["samples"]),
         "num_eval_candidates": len(eval_dataset["samples"]),
         "model_selection": best["selection"],
-        "fit_summary": _simple_summary(fit_cases),
-        "selection_summary": _simple_summary(select_cases),
-        "eval_summary": _simple_summary(eval_cases),
+        "fit_summary": _summary_with_pairwise(fit_cases, fit_pairwise),
+        "selection_summary": _summary_with_pairwise(select_cases, select_pairwise),
+        "eval_summary": _summary_with_pairwise(eval_cases, eval_pairwise),
+        "pairwise_causal_calibration": {
+            "fit": _pairwise_calibration_global(fit_pairwise),
+            "selection": _pairwise_calibration_global(select_pairwise),
+            "eval": _pairwise_calibration_global(eval_pairwise),
+        },
         "summary": summary,
         "rows": eval_cases,
     }
@@ -512,15 +541,16 @@ def _table(metrics: dict[str, Any]) -> str:
     summary = metrics["eval_summary"]
     lines = [
         "% Auto-generated by scripts/eval_nonlinear_dominance_selector.py",
-        "\\begin{tabular}{lrrrrrrrr}",
+        "\\begin{tabular}{lrrrrrrrrr}",
         "\\toprule",
-        "Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap \\\\",
+        "Rows & Coverage & Fallback & Base succ. & Selected succ. & Oracle succ. & OutcomePTR & Succ. support gap & Succ. selector gap & Cal. ECE \\\\",
         "\\midrule",
         f"{metrics['num_eval_rows']} & {_fmt(summary.get('coverage'))} & "
         f"{_fmt(summary.get('fallback_rate'))} & {_fmt(summary.get('base_success'))} & "
         f"{_fmt(summary.get('selected_success'))} & {_fmt(summary.get('proposal_oracle_success'))} & "
         f"{_fmt(summary.get('outcome_ptr'))} & {_fmt(summary.get('success_support_gap'))} & "
-        f"{_fmt(summary.get('success_selector_gap'))} \\\\",
+        f"{_fmt(summary.get('success_selector_gap'))} & "
+        f"{_fmt(summary.get('pairwise_causal_calibration_ece'))} \\\\",
         "\\bottomrule",
         "\\end{tabular}",
     ]
@@ -546,20 +576,23 @@ def _report(metrics: dict[str, Any]) -> str:
         "",
         "The model is fit on calibration-fit rows, and model/tau are selected on held-out calibration-selection rows only. Eval outcomes are used only for reporting.",
         "",
-        "| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap |",
-        "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+        "| Split | Coverage | Fallback | Base success | Selected success | Proposal oracle | OutcomePTR | Success support gap | Success selector gap | Calibration ECE |",
+        "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
         f"| fit | {_fmt(fit.get('coverage'))} | {_fmt(fit.get('fallback_rate'))} | "
         f"{_fmt(fit.get('base_success'))} | {_fmt(fit.get('selected_success'))} | "
         f"{_fmt(fit.get('proposal_oracle_success'))} | {_fmt(fit.get('outcome_ptr'))} | "
-        f"{_fmt(fit.get('success_support_gap'))} | {_fmt(fit.get('success_selector_gap'))} |",
+        f"{_fmt(fit.get('success_support_gap'))} | {_fmt(fit.get('success_selector_gap'))} | "
+        f"{_fmt(fit.get('pairwise_causal_calibration_ece'))} |",
         f"| selection | {_fmt(selection.get('coverage'))} | {_fmt(selection.get('fallback_rate'))} | "
         f"{_fmt(selection.get('base_success'))} | {_fmt(selection.get('selected_success'))} | "
         f"{_fmt(selection.get('proposal_oracle_success'))} | {_fmt(selection.get('outcome_ptr'))} | "
-        f"{_fmt(selection.get('success_support_gap'))} | {_fmt(selection.get('success_selector_gap'))} |",
+        f"{_fmt(selection.get('success_support_gap'))} | {_fmt(selection.get('success_selector_gap'))} | "
+        f"{_fmt(selection.get('pairwise_causal_calibration_ece'))} |",
         f"| eval | {_fmt(eval_summary.get('coverage'))} | {_fmt(eval_summary.get('fallback_rate'))} | "
         f"{_fmt(eval_summary.get('base_success'))} | {_fmt(eval_summary.get('selected_success'))} | "
         f"{_fmt(eval_summary.get('proposal_oracle_success'))} | {_fmt(eval_summary.get('outcome_ptr'))} | "
-        f"{_fmt(eval_summary.get('success_support_gap'))} | {_fmt(eval_summary.get('success_selector_gap'))} |",
+        f"{_fmt(eval_summary.get('success_support_gap'))} | {_fmt(eval_summary.get('success_selector_gap'))} | "
+        f"{_fmt(eval_summary.get('pairwise_causal_calibration_ece'))} |",
         "",
         "This is a train-calibrated selector diagnostic over already measured candidates, not a new rollout.",
     ]