Auto-sync: 2026-06-29 06:36:52

dovla_cil/eval/maniskill_policy_rollout.py

@@ -82,8 +82,10 @@ def evaluate_maniskill_policy_rollout(
     retrieval_metric: str = "raw",
     retrieval_type_min_success: float = 0.0,
     retrieval_residual_min_source_progress: float = 0.0,
+    retrieval_residual_min_source_advantage: float = -1.0e9,
     retrieval_residual_source_progress_bonus_scale: float = 0.0,
     retrieval_residual_source_score_bonus_scale: float = 0.0,
+    retrieval_residual_source_advantage_bonus_scale: float = 0.0,
     retrieval_residual_action_l2_penalty: float = 0.0,
     retrieval_residual_scale: float = 1.0,
     retrieval_residual_scales: tuple[float, ...] = (),
@@ -191,6 +193,8 @@ def evaluate_maniskill_policy_rollout(
         raise ValueError("retrieval_residual_source_progress_bonus_scale must be non-negative")
     if retrieval_residual_source_score_bonus_scale < 0:
         raise ValueError("retrieval_residual_source_score_bonus_scale must be non-negative")
+    if retrieval_residual_source_advantage_bonus_scale < 0:
+        raise ValueError("retrieval_residual_source_advantage_bonus_scale must be non-negative")
     if retrieval_residual_action_l2_penalty < 0:
         raise ValueError("retrieval_residual_action_l2_penalty must be non-negative")
     if retrieval_residual_scale < 0:
@@ -261,12 +265,18 @@ def evaluate_maniskill_policy_rollout(
                 retrieval_metric=retrieval_metric,
                 retrieval_type_min_success=retrieval_type_min_success,
                 retrieval_residual_min_source_progress=retrieval_residual_min_source_progress,
+                retrieval_residual_min_source_advantage=(
+                    retrieval_residual_min_source_advantage
+                ),
                 retrieval_residual_source_progress_bonus_scale=(
                     retrieval_residual_source_progress_bonus_scale
                 ),
                 retrieval_residual_source_score_bonus_scale=(
                     retrieval_residual_source_score_bonus_scale
                 ),
+                retrieval_residual_source_advantage_bonus_scale=(
+                    retrieval_residual_source_advantage_bonus_scale
+                ),
                 retrieval_residual_anchor=retrieval_residual_anchor,
                 retrieval_residual_reduce=retrieval_residual_reduce,
             )
@@ -358,6 +368,11 @@ def evaluate_maniskill_policy_rollout(
         "retrieval_residual_min_source_progress": retrieval_residual_min_source_progress
         if selection_mode == "retrieval_residual"
         else 0.0,
+        "retrieval_residual_min_source_advantage": (
+            retrieval_residual_min_source_advantage
+            if selection_mode == "retrieval_residual"
+            else -1.0e9
+        ),
         "retrieval_residual_source_progress_bonus_scale": (
             retrieval_residual_source_progress_bonus_scale
             if selection_mode == "retrieval_residual"
@@ -368,6 +383,11 @@ def evaluate_maniskill_policy_rollout(
             if selection_mode == "retrieval_residual"
             else 0.0
         ),
+        "retrieval_residual_source_advantage_bonus_scale": (
+            retrieval_residual_source_advantage_bonus_scale
+            if selection_mode == "retrieval_residual"
+            else 0.0
+        ),
         "retrieval_residual_action_l2_penalty": (
             retrieval_residual_action_l2_penalty
             if selection_mode == "retrieval_residual"
@@ -556,8 +576,10 @@ def _attach_retrieved_residual_candidates(
     retrieval_metric: str = "raw",
     retrieval_type_min_success: float = 0.0,
     retrieval_residual_min_source_progress: float = 0.0,
+    retrieval_residual_min_source_advantage: float = -1.0e9,
     retrieval_residual_source_progress_bonus_scale: float = 0.0,
     retrieval_residual_source_score_bonus_scale: float = 0.0,
+    retrieval_residual_source_advantage_bonus_scale: float = 0.0,
     retrieval_residual_anchor: str = "expert",
     retrieval_residual_reduce: str = "none",
 ) -> list[_RolloutCase]:
@@ -567,6 +589,7 @@ def _attach_retrieved_residual_candidates(
     uses_source_bonus = (
         retrieval_residual_source_progress_bonus_scale > 0
         or retrieval_residual_source_score_bonus_scale > 0
+        or retrieval_residual_source_advantage_bonus_scale > 0
     )
     type_success_rates = _candidate_type_success_rates(dataset, heldout_group_ids=heldout)
     bank: dict[
@@ -597,6 +620,9 @@ def _attach_retrieved_residual_candidates(
         if anchor is None:
             anchor = next((record for record in records if record.candidate_type == "expert"), records[0])
         anchor_action = np.asarray(_numeric_action_values(anchor), dtype=np.float32)
+        anchor_reward = getattr(anchor, "reward", None)
+        anchor_progress = float(getattr(anchor_reward, "progress", 0.0))
+        anchor_score = _source_reward_score(anchor_reward, progress=anchor_progress)
         residuals: list[list[list[float]]] = [np.zeros_like(anchor_action).tolist()]
         candidate_types = ["policy_residual"]
         residual_bonuses = [0.0]
@@ -609,14 +635,18 @@ def _attach_retrieved_residual_candidates(
             reward = getattr(record, "reward", None)
             source_progress = float(getattr(reward, "progress", 0.0))
             source_score = _source_reward_score(reward, progress=source_progress)
+            source_advantage = source_score - anchor_score
             if source_progress < retrieval_residual_min_source_progress:
                 continue
+            if source_advantage < retrieval_residual_min_source_advantage:
+                continue
             residual = np.asarray(_numeric_action_values(record), dtype=np.float32) - anchor_action
             residuals.append(residual.tolist())
             candidate_types.append(f"residual_{record.candidate_type}")
             residual_bonuses.append(
                 float(retrieval_residual_source_progress_bonus_scale) * source_progress
                 + float(retrieval_residual_source_score_bonus_scale) * source_score
+                + float(retrieval_residual_source_advantage_bonus_scale) * source_advantage
             )
         feature = np.asarray(
             vectorize_toy_observation(records[0].observation_inline or {}, obs_dim=obs_dim),

logs/auto_sync_hf.log

@@ -230,3 +230,4 @@ No files have been modified since last commit. Skipping to prevent empty commit.
 No files have been modified since last commit. Skipping to prevent empty commit.
 No files have been modified since last commit. Skipping to prevent empty commit.
 No files have been modified since last commit. Skipping to prevent empty commit.
+No files have been modified since last commit. Skipping to prevent empty commit.