Add learned dominance selector ablations

workspace/scripts/eval_learned_dominance_selector.py

@@ -21,7 +21,7 @@ from cil.metrics import macro_micro_summary  # noqa: E402
 from scripts.eval_dominance_selector import _DominanceScorer, _chart_map, _rows  # noqa: E402
 
 
-FEATURE_NAMES = [
+BASIC_FEATURE_NAMES = [
     "bias",
     "candidate_score",
     "candidate_score_minus_base_score",
@@ -33,6 +33,7 @@ FEATURE_NAMES = [
     "tangent_linf_norm",
     "num_candidates",
 ]
+FEATURE_NAMES = BASIC_FEATURE_NAMES
 
 
 def main(argv: list[str] | None = None) -> int:
@@ -53,6 +54,21 @@ def main(argv: list[str] | None = None) -> int:
     parser.add_argument("--out-dir", type=Path, default=Path("runs/ctt_learned_dominance_val_to_test"))
     parser.add_argument("--k", type=int, default=8)
     parser.add_argument("--ridge-lambdas", default="0,0.01,0.1,1,10,100")
+    parser.add_argument(
+        "--feature-set",
+        choices=("basic", "tangent"),
+        default="basic",
+        help="Deployment-visible feature family for candidate-level dominance fitting.",
+    )
+    parser.add_argument(
+        "--target",
+        choices=("utility_margin", "success", "success_weighted_margin"),
+        default="utility_margin",
+        help=(
+            "Calibration target. success_weighted_margin fits utility margin plus "
+            "candidate success to prioritize the lexicographic success/progress utility."
+        ),
+    )
     parser.add_argument("--bootstrap-samples", type=int, default=1000)
     args = parser.parse_args(argv)
 
@@ -77,12 +93,16 @@ def main(argv: list[str] | None = None) -> int:
         calibration_charts,
         scorer=scorer,
         k=args.k,
+        feature_set=args.feature_set,
+        target=args.target,
     )
     eval_dataset = _candidate_dataset(
         eval_rows,
         eval_charts,
         scorer=scorer,
         k=args.k,
+        feature_set=args.feature_set,
+        target=args.target,
     )
     best = _fit_select_ridge(calibration_dataset, lambdas=lambdas)
     eval_cases = _evaluate_dataset(eval_dataset, best["weights"], best["mean"], best["std"], tau=best["tau"])
@@ -116,7 +136,9 @@ def main(argv: list[str] | None = None) -> int:
         "report_type": "learned_dominance_selector_eval",
         "schema_version": 1,
         "k": args.k,
-        "feature_names": FEATURE_NAMES,
+        "feature_set": args.feature_set,
+        "target": args.target,
+        "feature_names": _feature_names(args.feature_set),
         "ridge_lambdas": lambdas,
         "selected_lambda": best["lambda"],
         "tau": best["tau"],
@@ -181,6 +203,8 @@ def _candidate_dataset(
     *,
     scorer: _DominanceScorer,
     k: int,
+    feature_set: str = "basic",
+    target: str = "utility_margin",
 ) -> dict[str, Any]:
     samples: list[dict[str, Any]] = []
     by_row: dict[int, list[int]] = defaultdict(list)
@@ -203,34 +227,34 @@ def _candidate_dataset(
                 tangents[candidate_index] if candidate_index < len(tangents) else [],
                 dtype=float,
             )
-            tangent_rms = float(np.linalg.norm(tangent) / math.sqrt(max(1, tangent.size)))
-            tangent_linf = float(np.max(np.abs(tangent))) if tangent.size else 0.0
-            feature = np.asarray(
-                [
-                    1.0,
-                    score,
-                    score - base_score,
-                    (score - score_mean) / score_std,
-                    float(candidate_index),
-                    float(candidate_index == 0),
-                    _source_rank(candidate_types[candidate_index] if candidate_index < len(candidate_types) else ""),
-                    tangent_rms,
-                    tangent_linf,
-                    float(len(scores)),
-                ],
-                dtype=float,
+            feature = _candidate_feature(
+                score=score,
+                base_score=base_score,
+                score_mean=score_mean,
+                score_std=score_std,
+                candidate_index=candidate_index,
+                candidate_type=candidate_types[candidate_index] if candidate_index < len(candidate_types) else "",
+                tangent=tangent,
+                num_candidates=len(scores),
+                feature_set=feature_set,
             )
             sample_index = len(samples)
             by_row[row_index].append(sample_index)
             base_utility = float(row["base_utility"])
             base_success = float(bool(row.get("base_success", False)))
             hidden = [float(value) for value in row.get("hidden_chart_utilities", [])]
+            target_margin = utilities[candidate_index] - base_utility
             samples.append(
                 {
                     "row_index": row_index,
                     "candidate_index": candidate_index,
                     "feature": feature,
-                    "target_margin": utilities[candidate_index] - base_utility,
+                    "target_margin": _target_value(
+                        target,
+                        utility_margin=target_margin,
+                        candidate_success=successes[candidate_index],
+                    ),
+                    "measured_utility_margin": target_margin,
                     "candidate_utility": utilities[candidate_index],
                     "candidate_success": successes[candidate_index],
                     "base_utility": base_utility,
@@ -249,6 +273,74 @@ def _candidate_dataset(
     return {"samples": samples, "by_row": dict(by_row), "num_rows": len(rows)}
 
 
+def _feature_names(feature_set: str) -> list[str]:
+    if feature_set == "basic":
+        return list(BASIC_FEATURE_NAMES)
+    if feature_set == "tangent":
+        return [
+            *BASIC_FEATURE_NAMES,
+            *[f"tangent_{index:02d}" for index in range(21)],
+            *[f"abs_tangent_{index:02d}" for index in range(21)],
+        ]
+    raise ValueError(f"unknown feature_set: {feature_set}")
+
+
+def _candidate_feature(
+    *,
+    score: float,
+    base_score: float,
+    score_mean: float,
+    score_std: float,
+    candidate_index: int,
+    candidate_type: Any,
+    tangent: np.ndarray,
+    num_candidates: int,
+    feature_set: str,
+) -> np.ndarray:
+    tangent = np.asarray(tangent, dtype=float).reshape(-1)
+    if tangent.size < 21:
+        tangent = np.pad(tangent, (0, 21 - tangent.size))
+    elif tangent.size > 21:
+        tangent = tangent[:21]
+    tangent_rms = float(np.linalg.norm(tangent) / math.sqrt(max(1, tangent.size)))
+    tangent_linf = float(np.max(np.abs(tangent))) if tangent.size else 0.0
+    basic = np.asarray(
+        [
+            1.0,
+            float(score),
+            float(score) - float(base_score),
+            (float(score) - float(score_mean)) / float(score_std),
+            float(candidate_index),
+            float(candidate_index == 0),
+            _source_rank(candidate_type),
+            tangent_rms,
+            tangent_linf,
+            float(num_candidates),
+        ],
+        dtype=float,
+    )
+    if feature_set == "basic":
+        return basic
+    if feature_set == "tangent":
+        return np.concatenate([basic, tangent.astype(float), np.abs(tangent).astype(float)])
+    raise ValueError(f"unknown feature_set: {feature_set}")
+
+
+def _target_value(
+    target: str,
+    *,
+    utility_margin: float,
+    candidate_success: float,
+) -> float:
+    if target == "utility_margin":
+        return float(utility_margin)
+    if target == "success":
+        return float(candidate_success)
+    if target == "success_weighted_margin":
+        return float(utility_margin) + float(candidate_success)
+    raise ValueError(f"unknown target: {target}")
+
+
 def _fit_select_ridge(dataset: dict[str, Any], *, lambdas: list[float]) -> dict[str, Any]:
     samples = dataset["samples"]
     if not samples:
@@ -460,6 +552,8 @@ def _report(metrics: dict[str, Any]) -> str:
         f"Eval rows: `{metrics['num_eval_rows']}`",
         f"Selected ridge lambda: `{metrics['selected_lambda']}`",
         f"Tau: `{metrics['tau']:.6f}`",
+        f"Feature set: `{metrics['feature_set']}`",
+        f"Target: `{metrics['target']}`",
         "",
         "The ridge calibrator and threshold are fit on calibration measured rows only. Eval outcomes are used only for reporting.",
         "",