sync dominance feature mode code 2026-07-03
Browse files
workspace/scripts/eval_dominance_selector.py
CHANGED
|
@@ -79,8 +79,15 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 79 |
calibrator = _DominanceScorer(args.checkpoint_template, score_source=args.score_source)
|
| 80 |
calibration_rows = _rows(json.loads(args.calibration_input.read_text()))
|
| 81 |
eval_rows = _rows(json.loads(args.eval_input.read_text()))
|
| 82 |
-
|
| 83 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 84 |
|
| 85 |
calibration_cases = [
|
| 86 |
_dominance_case(row, calibration_charts, scorer=calibrator, k=args.k)
|
|
@@ -135,6 +142,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
| 135 |
"tau": tau,
|
| 136 |
"tau_mode": args.tau,
|
| 137 |
"score_source": args.score_source,
|
|
|
|
| 138 |
"checkpoint_template": args.checkpoint_template,
|
| 139 |
"residual_quantile": residual_quantile,
|
| 140 |
"calibration_input": str(args.calibration_input),
|
|
@@ -183,9 +191,16 @@ class _DominanceScorer:
|
|
| 183 |
self.checkpoint_template = checkpoint_template
|
| 184 |
self.score_source = score_source
|
| 185 |
self._models: dict[str, tuple[ChartEncoder, UtilityEnergy, TangentNormalizer, int]] = {}
|
|
|
|
| 186 |
self._encoded_chart_cache: dict[tuple[str, str], torch.Tensor] = {}
|
| 187 |
self._base_score_cache: dict[tuple[str, str], float] = {}
|
| 188 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 189 |
def base_score(self, row: dict[str, Any], chart: Any) -> float:
|
| 190 |
if "base_predicted_score" in row:
|
| 191 |
return float(row["base_predicted_score"])
|
|
@@ -249,6 +264,7 @@ class _DominanceScorer:
|
|
| 249 |
chart_feature_dim = int(checkpoint["feature_dim"])
|
| 250 |
chart_dim = int(checkpoint.get("chart_dim", 64))
|
| 251 |
tangent_dim = int(checkpoint["tangent_dim"])
|
|
|
|
| 252 |
encoder = ChartEncoder(chart_feature_dim, output_dim=chart_dim)
|
| 253 |
utility_energy = UtilityEnergy(chart_dim=chart_dim, tangent_dim=tangent_dim)
|
| 254 |
encoder.load_state_dict(checkpoint["chart_encoder"])
|
|
@@ -380,17 +396,25 @@ def _conformal_quantile(values: list[float], *, alpha: float) -> float:
|
|
| 380 |
return clean[index]
|
| 381 |
|
| 382 |
|
| 383 |
-
def _chart_map(index_path: Path) -> tuple[dict[str, Any], dict[str, Any]]:
|
| 384 |
charts, index = load_chart_items(
|
| 385 |
index_path,
|
| 386 |
max_charts=None,
|
| 387 |
require_positive=True,
|
| 388 |
include_hidden=True,
|
| 389 |
include_metadata=True,
|
|
|
|
| 390 |
)
|
| 391 |
return {chart.chart_id: chart for chart in charts}, index
|
| 392 |
|
| 393 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 394 |
def _rows(payload: Any) -> list[dict[str, Any]]:
|
| 395 |
rows = payload.get("rows", payload) if isinstance(payload, dict) else payload
|
| 396 |
if not isinstance(rows, list):
|
|
|
|
| 79 |
calibrator = _DominanceScorer(args.checkpoint_template, score_source=args.score_source)
|
| 80 |
calibration_rows = _rows(json.loads(args.calibration_input.read_text()))
|
| 81 |
eval_rows = _rows(json.loads(args.eval_input.read_text()))
|
| 82 |
+
chart_feature_mode = calibrator.chart_feature_mode(_first_train_seed(calibration_rows + eval_rows))
|
| 83 |
+
calibration_charts, calibration_index = _chart_map(
|
| 84 |
+
args.calibration_target_index,
|
| 85 |
+
chart_feature_mode=chart_feature_mode,
|
| 86 |
+
)
|
| 87 |
+
eval_charts, eval_index = _chart_map(
|
| 88 |
+
args.eval_target_index,
|
| 89 |
+
chart_feature_mode=chart_feature_mode,
|
| 90 |
+
)
|
| 91 |
|
| 92 |
calibration_cases = [
|
| 93 |
_dominance_case(row, calibration_charts, scorer=calibrator, k=args.k)
|
|
|
|
| 142 |
"tau": tau,
|
| 143 |
"tau_mode": args.tau,
|
| 144 |
"score_source": args.score_source,
|
| 145 |
+
"chart_feature_mode": chart_feature_mode,
|
| 146 |
"checkpoint_template": args.checkpoint_template,
|
| 147 |
"residual_quantile": residual_quantile,
|
| 148 |
"calibration_input": str(args.calibration_input),
|
|
|
|
| 191 |
self.checkpoint_template = checkpoint_template
|
| 192 |
self.score_source = score_source
|
| 193 |
self._models: dict[str, tuple[ChartEncoder, UtilityEnergy, TangentNormalizer, int]] = {}
|
| 194 |
+
self._feature_modes: dict[str, str] = {}
|
| 195 |
self._encoded_chart_cache: dict[tuple[str, str], torch.Tensor] = {}
|
| 196 |
self._base_score_cache: dict[tuple[str, str], float] = {}
|
| 197 |
|
| 198 |
+
def chart_feature_mode(self, seed: str) -> str:
|
| 199 |
+
if self.score_source == "row":
|
| 200 |
+
return "base"
|
| 201 |
+
self._model(seed)
|
| 202 |
+
return self._feature_modes.get(seed, "base")
|
| 203 |
+
|
| 204 |
def base_score(self, row: dict[str, Any], chart: Any) -> float:
|
| 205 |
if "base_predicted_score" in row:
|
| 206 |
return float(row["base_predicted_score"])
|
|
|
|
| 264 |
chart_feature_dim = int(checkpoint["feature_dim"])
|
| 265 |
chart_dim = int(checkpoint.get("chart_dim", 64))
|
| 266 |
tangent_dim = int(checkpoint["tangent_dim"])
|
| 267 |
+
self._feature_modes[seed] = str(checkpoint.get("chart_feature_mode", "base"))
|
| 268 |
encoder = ChartEncoder(chart_feature_dim, output_dim=chart_dim)
|
| 269 |
utility_energy = UtilityEnergy(chart_dim=chart_dim, tangent_dim=tangent_dim)
|
| 270 |
encoder.load_state_dict(checkpoint["chart_encoder"])
|
|
|
|
| 396 |
return clean[index]
|
| 397 |
|
| 398 |
|
| 399 |
+
def _chart_map(index_path: Path, *, chart_feature_mode: str = "base") -> tuple[dict[str, Any], dict[str, Any]]:
|
| 400 |
charts, index = load_chart_items(
|
| 401 |
index_path,
|
| 402 |
max_charts=None,
|
| 403 |
require_positive=True,
|
| 404 |
include_hidden=True,
|
| 405 |
include_metadata=True,
|
| 406 |
+
chart_feature_mode=chart_feature_mode,
|
| 407 |
)
|
| 408 |
return {chart.chart_id: chart for chart in charts}, index
|
| 409 |
|
| 410 |
|
| 411 |
+
def _first_train_seed(rows: list[dict[str, Any]]) -> str:
|
| 412 |
+
for row in rows:
|
| 413 |
+
if row.get("train_seed") is not None:
|
| 414 |
+
return str(row.get("train_seed"))
|
| 415 |
+
return "0"
|
| 416 |
+
|
| 417 |
+
|
| 418 |
def _rows(payload: Any) -> list[dict[str, Any]]:
|
| 419 |
rows = payload.get("rows", payload) if isinstance(payload, dict) else payload
|
| 420 |
if not isinstance(rows, list):
|