anhtld commited on
Commit
6daff59
·
verified ·
1 Parent(s): 87930a8

sync dominance feature mode code 2026-07-03

Browse files
workspace/scripts/eval_dominance_selector.py CHANGED
@@ -79,8 +79,15 @@ def main(argv: list[str] | None = None) -> int:
79
  calibrator = _DominanceScorer(args.checkpoint_template, score_source=args.score_source)
80
  calibration_rows = _rows(json.loads(args.calibration_input.read_text()))
81
  eval_rows = _rows(json.loads(args.eval_input.read_text()))
82
- calibration_charts, calibration_index = _chart_map(args.calibration_target_index)
83
- eval_charts, eval_index = _chart_map(args.eval_target_index)
 
 
 
 
 
 
 
84
 
85
  calibration_cases = [
86
  _dominance_case(row, calibration_charts, scorer=calibrator, k=args.k)
@@ -135,6 +142,7 @@ def main(argv: list[str] | None = None) -> int:
135
  "tau": tau,
136
  "tau_mode": args.tau,
137
  "score_source": args.score_source,
 
138
  "checkpoint_template": args.checkpoint_template,
139
  "residual_quantile": residual_quantile,
140
  "calibration_input": str(args.calibration_input),
@@ -183,9 +191,16 @@ class _DominanceScorer:
183
  self.checkpoint_template = checkpoint_template
184
  self.score_source = score_source
185
  self._models: dict[str, tuple[ChartEncoder, UtilityEnergy, TangentNormalizer, int]] = {}
 
186
  self._encoded_chart_cache: dict[tuple[str, str], torch.Tensor] = {}
187
  self._base_score_cache: dict[tuple[str, str], float] = {}
188
 
 
 
 
 
 
 
189
  def base_score(self, row: dict[str, Any], chart: Any) -> float:
190
  if "base_predicted_score" in row:
191
  return float(row["base_predicted_score"])
@@ -249,6 +264,7 @@ class _DominanceScorer:
249
  chart_feature_dim = int(checkpoint["feature_dim"])
250
  chart_dim = int(checkpoint.get("chart_dim", 64))
251
  tangent_dim = int(checkpoint["tangent_dim"])
 
252
  encoder = ChartEncoder(chart_feature_dim, output_dim=chart_dim)
253
  utility_energy = UtilityEnergy(chart_dim=chart_dim, tangent_dim=tangent_dim)
254
  encoder.load_state_dict(checkpoint["chart_encoder"])
@@ -380,17 +396,25 @@ def _conformal_quantile(values: list[float], *, alpha: float) -> float:
380
  return clean[index]
381
 
382
 
383
- def _chart_map(index_path: Path) -> tuple[dict[str, Any], dict[str, Any]]:
384
  charts, index = load_chart_items(
385
  index_path,
386
  max_charts=None,
387
  require_positive=True,
388
  include_hidden=True,
389
  include_metadata=True,
 
390
  )
391
  return {chart.chart_id: chart for chart in charts}, index
392
 
393
 
 
 
 
 
 
 
 
394
  def _rows(payload: Any) -> list[dict[str, Any]]:
395
  rows = payload.get("rows", payload) if isinstance(payload, dict) else payload
396
  if not isinstance(rows, list):
 
79
  calibrator = _DominanceScorer(args.checkpoint_template, score_source=args.score_source)
80
  calibration_rows = _rows(json.loads(args.calibration_input.read_text()))
81
  eval_rows = _rows(json.loads(args.eval_input.read_text()))
82
+ chart_feature_mode = calibrator.chart_feature_mode(_first_train_seed(calibration_rows + eval_rows))
83
+ calibration_charts, calibration_index = _chart_map(
84
+ args.calibration_target_index,
85
+ chart_feature_mode=chart_feature_mode,
86
+ )
87
+ eval_charts, eval_index = _chart_map(
88
+ args.eval_target_index,
89
+ chart_feature_mode=chart_feature_mode,
90
+ )
91
 
92
  calibration_cases = [
93
  _dominance_case(row, calibration_charts, scorer=calibrator, k=args.k)
 
142
  "tau": tau,
143
  "tau_mode": args.tau,
144
  "score_source": args.score_source,
145
+ "chart_feature_mode": chart_feature_mode,
146
  "checkpoint_template": args.checkpoint_template,
147
  "residual_quantile": residual_quantile,
148
  "calibration_input": str(args.calibration_input),
 
191
  self.checkpoint_template = checkpoint_template
192
  self.score_source = score_source
193
  self._models: dict[str, tuple[ChartEncoder, UtilityEnergy, TangentNormalizer, int]] = {}
194
+ self._feature_modes: dict[str, str] = {}
195
  self._encoded_chart_cache: dict[tuple[str, str], torch.Tensor] = {}
196
  self._base_score_cache: dict[tuple[str, str], float] = {}
197
 
198
+ def chart_feature_mode(self, seed: str) -> str:
199
+ if self.score_source == "row":
200
+ return "base"
201
+ self._model(seed)
202
+ return self._feature_modes.get(seed, "base")
203
+
204
  def base_score(self, row: dict[str, Any], chart: Any) -> float:
205
  if "base_predicted_score" in row:
206
  return float(row["base_predicted_score"])
 
264
  chart_feature_dim = int(checkpoint["feature_dim"])
265
  chart_dim = int(checkpoint.get("chart_dim", 64))
266
  tangent_dim = int(checkpoint["tangent_dim"])
267
+ self._feature_modes[seed] = str(checkpoint.get("chart_feature_mode", "base"))
268
  encoder = ChartEncoder(chart_feature_dim, output_dim=chart_dim)
269
  utility_energy = UtilityEnergy(chart_dim=chart_dim, tangent_dim=tangent_dim)
270
  encoder.load_state_dict(checkpoint["chart_encoder"])
 
396
  return clean[index]
397
 
398
 
399
+ def _chart_map(index_path: Path, *, chart_feature_mode: str = "base") -> tuple[dict[str, Any], dict[str, Any]]:
400
  charts, index = load_chart_items(
401
  index_path,
402
  max_charts=None,
403
  require_positive=True,
404
  include_hidden=True,
405
  include_metadata=True,
406
+ chart_feature_mode=chart_feature_mode,
407
  )
408
  return {chart.chart_id: chart for chart in charts}, index
409
 
410
 
411
+ def _first_train_seed(rows: list[dict[str, Any]]) -> str:
412
+ for row in rows:
413
+ if row.get("train_seed") is not None:
414
+ return str(row.get("train_seed"))
415
+ return "0"
416
+
417
+
418
  def _rows(payload: Any) -> list[dict[str, Any]]:
419
  rows = payload.get("rows", payload) if isinstance(payload, dict) else payload
420
  if not isinstance(rows, list):