auto-sync 2026-07-04T01:14:16Z workspace (part 3)

workspace/scripts/eval_learned_dominance_selector.py

@@ -18,8 +18,10 @@ if str(PROJECT_ROOT) not in sys.path:
 
 import numpy as np  # noqa: E402
 
+from cil.chart_features import CHART_FEATURE_MODES, OBJECT_LAYOUT_EMBED_DIM, OBSERVATION_EMBED_DIM  # noqa: E402
 from cil.metrics import macro_micro_summary  # noqa: E402
 from scripts.eval_dominance_selector import _DominanceScorer, _chart_map, _first_train_seed, _rows  # noqa: E402
+from scripts.eval_ctt_generated_rollout import load_chart_items  # noqa: E402
 
 
 BASIC_FEATURE_NAMES = [
@@ -56,6 +58,30 @@ SOURCE_EVIDENCE_NAMES = [
     "generated_to_source_negative_min_rms",
     "generated_source_pos_closer_than_neg",
 ]
+CHART_COMPAT_NAMES = [
+    "source_chart_feature_found",
+    "target_chart_feature_norm",
+    "source_chart_feature_norm",
+    "target_source_chart_cosine",
+    "target_source_chart_rms",
+    "target_source_chart_linf",
+    "target_source_chart_absmean",
+    "target_source_chart_dot_per_dim",
+    "target_base_action_norm",
+    "source_base_action_norm",
+    "target_source_base_cosine",
+    "target_source_base_rms",
+    "target_source_obs_available",
+    "target_source_obs_cosine",
+    "target_source_obs_rms",
+    "target_obs_norm",
+    "source_obs_norm",
+    "target_source_obj_available",
+    "target_source_obj_cosine",
+    "target_source_obj_rms",
+    "target_obj_norm",
+    "source_obj_norm",
+]
 FEATURE_SET_CHOICES = (
     "basic",
     "tangent",
@@ -65,6 +91,10 @@ FEATURE_SET_CHOICES = (
     "tangent_source_evidence",
     "context_source_evidence",
     "context_tangent_source_evidence",
+    "chart_compat",
+    "chart_tangent_compat",
+    "chart_source_compat",
+    "chart_tangent_source_compat",
 )
 
 
@@ -101,6 +131,15 @@ def main(argv: list[str] | None = None) -> int:
         default="basic",
         help="Deployment-visible feature family for candidate-level dominance fitting.",
     )
+    parser.add_argument(
+        "--selector-chart-feature-mode",
+        choices=CHART_FEATURE_MODES,
+        default="base_context_obs_obj",
+        help=(
+            "Chart feature mode used only for selector chart-compatibility "
+            "features. These maps are loaded without hidden outcomes."
+        ),
+    )
     parser.add_argument(
         "--target",
         choices=("utility_margin", "success", "success_weighted_margin"),
@@ -166,6 +205,25 @@ def main(argv: list[str] | None = None) -> int:
     source_evidence, source_index = (
         _source_evidence_map(source_index_path) if _uses_source_evidence(args.feature_set) else ({}, {})
     )
+    selector_source_charts: dict[str, Any] = {}
+    selector_source_index: dict[str, Any] = {}
+    selector_calibration_charts: dict[str, Any] = {}
+    selector_eval_charts: dict[str, Any] = {}
+    selector_calibration_index: dict[str, Any] = {}
+    selector_eval_index: dict[str, Any] = {}
+    if _uses_chart_compat(args.feature_set):
+        selector_calibration_charts, selector_calibration_index = _selector_chart_map(
+            args.calibration_target_index,
+            chart_feature_mode=args.selector_chart_feature_mode,
+        )
+        selector_eval_charts, selector_eval_index = _selector_chart_map(
+            args.eval_target_index,
+            chart_feature_mode=args.selector_chart_feature_mode,
+        )
+        selector_source_charts, selector_source_index = _selector_chart_map(
+            source_index_path,
+            chart_feature_mode=args.selector_chart_feature_mode,
+        )
 
     calibration_dataset = _candidate_dataset(
         calibration_rows,
@@ -175,6 +233,9 @@ def main(argv: list[str] | None = None) -> int:
         feature_set=args.feature_set,
         target=args.target,
         source_evidence=source_evidence,
+        selector_target_charts=selector_calibration_charts,
+        selector_source_charts=selector_source_charts,
+        selector_chart_feature_mode=args.selector_chart_feature_mode,
     )
     eval_dataset = _candidate_dataset(
         eval_rows,
@@ -184,6 +245,9 @@ def main(argv: list[str] | None = None) -> int:
         feature_set=args.feature_set,
         target=args.target,
         source_evidence=source_evidence,
+        selector_target_charts=selector_eval_charts,
+        selector_source_charts=selector_source_charts,
+        selector_chart_feature_mode=args.selector_chart_feature_mode,
     )
     best = _fit_select_ridge(
         calibration_dataset,
@@ -224,6 +288,9 @@ def main(argv: list[str] | None = None) -> int:
         "schema_version": 1,
         "k": args.k,
         "feature_set": args.feature_set,
+        "selector_chart_feature_mode": (
+            args.selector_chart_feature_mode if _uses_chart_compat(args.feature_set) else None
+        ),
         "target": args.target,
         "fit_objective": args.fit_objective,
         "pairwise_weight": args.pairwise_weight,
@@ -248,6 +315,12 @@ def main(argv: list[str] | None = None) -> int:
         "eval_target_split_hash": eval_index.get("split_hash"),
         "source_content_hash": source_index.get("content_hash"),
         "source_split_hash": source_index.get("split_hash"),
+        "selector_source_content_hash": selector_source_index.get("content_hash"),
+        "selector_source_split_hash": selector_source_index.get("split_hash"),
+        "selector_calibration_target_content_hash": selector_calibration_index.get("content_hash"),
+        "selector_calibration_target_split_hash": selector_calibration_index.get("split_hash"),
+        "selector_eval_target_content_hash": selector_eval_index.get("content_hash"),
+        "selector_eval_target_split_hash": selector_eval_index.get("split_hash"),
         "num_calibration_rows": len(calibration_rows),
         "num_eval_rows": len(eval_rows),
         "num_calibration_candidates": len(calibration_dataset["samples"]),
@@ -304,14 +377,20 @@ def _candidate_dataset(
     feature_set: str = "basic",
     target: str = "utility_margin",
     source_evidence: dict[str, dict[str, Any]] | None = None,
+    selector_target_charts: dict[str, Any] | None = None,
+    selector_source_charts: dict[str, Any] | None = None,
+    selector_chart_feature_mode: str = "base_context_obs_obj",
 ) -> dict[str, Any]:
     source_evidence = source_evidence or {}
+    selector_target_charts = selector_target_charts or {}
+    selector_source_charts = selector_source_charts or {}
     samples: list[dict[str, Any]] = []
     by_row: dict[int, list[int]] = defaultdict(list)
     for row_index, row in enumerate(rows):
         chart_id = str(row.get("chart_id", row.get("group_id", "")))
         if chart_id not in charts:
             raise KeyError(f"chart_id {chart_id!r} not found in target index")
+        selector_target_chart = selector_target_charts.get(chart_id)
         base_score = scorer.base_score(row, charts[chart_id])
         scores = [float(value) for value in row.get("predicted_scores", [])[:k]]
         utilities = [float(value) for value in row.get("generated_utilities", [])[:k]]
@@ -325,6 +404,7 @@ def _candidate_dataset(
         score_mean = sum(scores) / len(scores)
         score_std = math.sqrt(sum((score - score_mean) ** 2 for score in scores) / len(scores)) + 1.0e-6
         for candidate_index, score in enumerate(scores):
+            source_chart_id = str(source_chart_ids[candidate_index]) if candidate_index < len(source_chart_ids) else ""
             tangent = np.asarray(
                 tangents[candidate_index] if candidate_index < len(tangents) else [],
                 dtype=float,
@@ -346,12 +426,15 @@ def _candidate_dataset(
                 tangent=tangent,
                 source_evidence=_source_evidence_feature(
                     source_evidence.get(
-                        str(source_chart_ids[candidate_index])
-                        if candidate_index < len(source_chart_ids)
-                        else ""
+                        source_chart_id
                     ),
                     tangent=tangent,
                 ),
+                chart_compat=_chart_compat_feature(
+                    selector_target_chart,
+                    selector_source_charts.get(source_chart_id),
+                    chart_feature_mode=selector_chart_feature_mode,
+                ),
                 num_candidates=len(scores),
                 feature_set=feature_set,
             )
@@ -412,6 +495,8 @@ def _feature_names(feature_set: str) -> list[str]:
         names.extend(tangent_names)
     if _uses_source_evidence(feature_set):
         names.extend(SOURCE_EVIDENCE_NAMES)
+    if _uses_chart_compat(feature_set):
+        names.extend(CHART_COMPAT_NAMES)
     if feature_set in FEATURE_SET_CHOICES:
         return names
     raise ValueError(f"unknown feature_set: {feature_set}")
@@ -430,6 +515,7 @@ def _candidate_feature(
     feature_set: str,
     context: dict[str, Any] | None = None,
     source_evidence: np.ndarray | None = None,
+    chart_compat: np.ndarray | None = None,
 ) -> np.ndarray:
     tangent = np.asarray(tangent, dtype=float).reshape(-1)
     if tangent.size < 21:
@@ -464,6 +550,10 @@ def _candidate_feature(
         if source_evidence is None:
             source_evidence = np.zeros(len(SOURCE_EVIDENCE_NAMES), dtype=float)
         parts.append(np.asarray(source_evidence, dtype=float).reshape(-1))
+    if _uses_chart_compat(feature_set):
+        if chart_compat is None:
+            chart_compat = np.zeros(len(CHART_COMPAT_NAMES), dtype=float)
+        parts.append(np.asarray(chart_compat, dtype=float).reshape(-1))
     if feature_set in FEATURE_SET_CHOICES:
         return np.concatenate(parts)
     raise ValueError(f"unknown feature_set: {feature_set}")
@@ -484,6 +574,8 @@ def _uses_tangent(feature_set: str) -> bool:
         "context_tangent",
         "tangent_source_evidence",
         "context_tangent_source_evidence",
+        "chart_tangent_compat",
+        "chart_tangent_source_compat",
     }
 
 
@@ -493,9 +585,32 @@ def _uses_source_evidence(feature_set: str) -> bool:
         "tangent_source_evidence",
         "context_source_evidence",
         "context_tangent_source_evidence",
+        "chart_source_compat",
+        "chart_tangent_source_compat",
     }
 
 
+def _uses_chart_compat(feature_set: str) -> bool:
+    return feature_set in {
+        "chart_compat",
+        "chart_tangent_compat",
+        "chart_source_compat",
+        "chart_tangent_source_compat",
+    }
+
+
+def _selector_chart_map(index_path: Path, *, chart_feature_mode: str) -> tuple[dict[str, Any], dict[str, Any]]:
+    charts, index = load_chart_items(
+        _resolve_index_path(index_path),
+        max_charts=None,
+        require_positive=False,
+        include_hidden=False,
+        include_metadata=True,
+        chart_feature_mode=chart_feature_mode,
+    )
+    return {chart.chart_id: chart for chart in charts}, index
+
+
 def _source_evidence_map(index_path: Path) -> tuple[dict[str, dict[str, Any]], dict[str, Any]]:
     index_path = _resolve_index_path(index_path)
     index = json.loads(index_path.read_text())
@@ -590,6 +705,126 @@ def _source_evidence_feature(source: dict[str, Any] | None, *, tangent: np.ndarr
     )
 
 
+def _chart_compat_feature(
+    target_chart: Any | None,
+    source_chart: Any | None,
+    *,
+    chart_feature_mode: str,
+) -> np.ndarray:
+    if target_chart is None or source_chart is None:
+        return np.zeros(len(CHART_COMPAT_NAMES), dtype=float)
+    target_feature = np.asarray(getattr(target_chart, "feature", []), dtype=float).reshape(-1)
+    source_feature = np.asarray(getattr(source_chart, "feature", []), dtype=float).reshape(-1)
+    width = min(target_feature.size, source_feature.size)
+    if width == 0:
+        return np.zeros(len(CHART_COMPAT_NAMES), dtype=float)
+    target_feature = target_feature[:width]
+    source_feature = source_feature[:width]
+    feature_diff = target_feature - source_feature
+
+    target_base = np.asarray(getattr(target_chart, "base_action", []), dtype=float).reshape(-1)
+    source_base = np.asarray(getattr(source_chart, "base_action", []), dtype=float).reshape(-1)
+    base_width = min(target_base.size, source_base.size)
+    target_base = target_base[:base_width]
+    source_base = source_base[:base_width]
+    base_diff = target_base - source_base if base_width else np.asarray([], dtype=float)
+
+    target_segments = _chart_feature_segments(target_chart, chart_feature_mode=chart_feature_mode)
+    source_segments = _chart_feature_segments(source_chart, chart_feature_mode=chart_feature_mode)
+    target_obs, source_obs = target_segments.get("obs"), source_segments.get("obs")
+    target_obj, source_obj = target_segments.get("obj"), source_segments.get("obj")
+    obs_available = float(target_obs is not None and source_obs is not None)
+    obj_available = float(target_obj is not None and source_obj is not None)
+    obs_diff = (
+        np.asarray(target_obs, dtype=float) - np.asarray(source_obs, dtype=float)
+        if obs_available
+        else np.asarray([], dtype=float)
+    )
+    obj_diff = (
+        np.asarray(target_obj, dtype=float) - np.asarray(source_obj, dtype=float)
+        if obj_available
+        else np.asarray([], dtype=float)
+    )
+
+    return np.asarray(
+        [
+            1.0,
+            _rms(target_feature),
+            _rms(source_feature),
+            _cosine(target_feature, source_feature),
+            _rms(feature_diff),
+            _linf(feature_diff),
+            _absmean(feature_diff),
+            float(np.dot(target_feature, source_feature) / max(1, width)),
+            _rms(target_base),
+            _rms(source_base),
+            _cosine(target_base, source_base),
+            _rms(base_diff),
+            obs_available,
+            _cosine(target_obs, source_obs) if obs_available else 0.0,
+            _rms(obs_diff),
+            _rms(target_obs) if obs_available else 0.0,
+            _rms(source_obs) if obs_available else 0.0,
+            obj_available,
+            _cosine(target_obj, source_obj) if obj_available else 0.0,
+            _rms(obj_diff),
+            _rms(target_obj) if obj_available else 0.0,
+            _rms(source_obj) if obj_available else 0.0,
+        ],
+        dtype=float,
+    )
+
+
+def _chart_feature_segments(chart: Any, *, chart_feature_mode: str) -> dict[str, np.ndarray]:
+    feature = np.asarray(getattr(chart, "feature", []), dtype=float).reshape(-1)
+    base_action = np.asarray(getattr(chart, "base_action", []), dtype=float).reshape(-1)
+    offset = min(base_action.size, feature.size)
+    if chart_feature_mode == "base":
+        return {}
+    context_width = 2 * CONTEXT_HASH_WIDTH + 2
+    offset = min(feature.size, offset + context_width)
+    segments: dict[str, np.ndarray] = {}
+    if chart_feature_mode in {"base_context_obs", "base_context_obs_obj"}:
+        end = min(feature.size, offset + OBSERVATION_EMBED_DIM)
+        if end - offset == OBSERVATION_EMBED_DIM:
+            segments["obs"] = feature[offset:end]
+        offset = end
+    if chart_feature_mode in {"base_context_obj", "base_context_obs_obj"}:
+        end = min(feature.size, offset + OBJECT_LAYOUT_EMBED_DIM)
+        if end - offset == OBJECT_LAYOUT_EMBED_DIM:
+            segments["obj"] = feature[offset:end]
+    return segments
+
+
+def _rms(values: Any) -> float:
+    array = np.asarray(values, dtype=float).reshape(-1)
+    return float(np.sqrt(np.mean(array * array))) if array.size else 0.0
+
+
+def _linf(values: Any) -> float:
+    array = np.asarray(values, dtype=float).reshape(-1)
+    return float(np.max(np.abs(array))) if array.size else 0.0
+
+
+def _absmean(values: Any) -> float:
+    array = np.asarray(values, dtype=float).reshape(-1)
+    return float(np.mean(np.abs(array))) if array.size else 0.0
+
+
+def _cosine(left: Any, right: Any) -> float:
+    left_array = np.asarray(left, dtype=float).reshape(-1)
+    right_array = np.asarray(right, dtype=float).reshape(-1)
+    width = min(left_array.size, right_array.size)
+    if width == 0:
+        return 0.0
+    left_array = left_array[:width]
+    right_array = right_array[:width]
+    denom = float(np.linalg.norm(left_array) * np.linalg.norm(right_array))
+    if denom <= 1.0e-12:
+        return 0.0
+    return float(np.dot(left_array, right_array) / denom)
+
+
 def _clean_array(values: Any) -> np.ndarray:
     array = np.asarray(values, dtype=float).reshape(-1)
     return array[np.isfinite(array)]
@@ -1080,8 +1315,10 @@ def _write_provenance(out_dir: Path, args: argparse.Namespace) -> None:
         "eval_input": _sha256(args.eval_input),
         "eval_target_index": _sha256(_resolve_index_path(args.eval_target_index)),
     }
-    if getattr(args, "source_index", None) is not None:
-        hashes["source_index"] = _sha256(_resolve_index_path(args.source_index))
+    if getattr(args, "source_index", None) is not None or _uses_chart_compat(args.feature_set):
+        hashes["source_index"] = _sha256(
+            _resolve_index_path(args.source_index or args.calibration_target_index)
+        )
     (out_dir / "data_hash.txt").write_text(json.dumps(hashes, indent=2, sort_keys=True) + "\n")
     (out_dir / "split_hash.txt").write_text(
         json.dumps(