feat(grade): inspected is upgraded to inspected_order. It rewards steps taken in order

server/WhyDidItFail_environment.py

@@ -26,7 +26,7 @@ class WhyDidItFailEnvironment(Environment):
     def __init__(self):
         self._state = State(episode_id=str(uuid4()), step_count=0)
         self.scenario: dict | None = None
-        self.inspected: set[str] = set()
+        self.inspection_order: list[str] = []  # first-visit order; doubles as membership check
 
     @property
     def state(self) -> State:
@@ -34,7 +34,7 @@ class WhyDidItFailEnvironment(Environment):
 
     def reset(self, seed: Optional[int] = None, episode_id: Optional[str] = None, **kwargs: Any) -> WhyDidItFailObservation:
         self._state = State(episode_id=episode_id or str(uuid4()), step_count=0)
-        self.inspected = set()
+        self.inspection_order = []
 
         scenario_key = kwargs.get("scenario_key")
         if scenario_key and scenario_key in SCENARIOS:
@@ -66,7 +66,8 @@ class WhyDidItFailEnvironment(Environment):
 
         if action.action_type == "inspect_logs":
             step_reward = self._inspect_reward("logs", required)
-            self.inspected.add("logs")
+            if "logs" not in self.inspection_order:
+                self.inspection_order.append("logs")
             return WhyDidItFailObservation(
                 task_description="Continue your investigation.",
                 visible_data={"training_logs": self.scenario["logs"]},
@@ -79,7 +80,8 @@ class WhyDidItFailEnvironment(Environment):
 
         elif action.action_type == "inspect_config":
             step_reward = self._inspect_reward("config", required)
-            self.inspected.add("config")
+            if "config" not in self.inspection_order:
+                self.inspection_order.append("config")
             return WhyDidItFailObservation(
                 task_description="Continue your investigation.",
                 visible_data={"config": self.scenario["config"]},
@@ -92,7 +94,8 @@ class WhyDidItFailEnvironment(Environment):
 
         elif action.action_type == "inspect_gradients":
             step_reward = self._inspect_reward("gradients", required)
-            self.inspected.add("gradients")
+            if "gradients" not in self.inspection_order:
+                self.inspection_order.append("gradients")
             return WhyDidItFailObservation(
                 task_description="Continue your investigation.",
                 visible_data={"gradient_norms": self.scenario["gradient_norms"]},
@@ -128,35 +131,51 @@ class WhyDidItFailEnvironment(Environment):
 
     # ── helpers ──────────────────────────────────────────────────────────────
 
+    # Rewards decay as more required sources are discovered — first clue is worth most.
+    _REQUIRED_STEP_REWARDS = [0.10, 0.07, 0.05]
+
     def _inspect_reward(self, source: str, required: list[str]) -> float:
-        """Return step reward for an inspect action."""
-        if source in self.inspected:
+        """Return step reward for an inspect action.
+
+        Required sources:   progressive — +0.10 / +0.07 / +0.05 for 1st/2nd/3rd discovery.
+        Irrelevant sources: -0.03 (mild; some exploration is acceptable).
+        Re-inspection:      -0.05 (waste).
+        """
+        if source in self.inspection_order:
             return -0.05   # redundant inspection
+
         if source in required:
-            return +0.05   # useful evidence
-        return -0.05       # irrelevant source
+            n_found = sum(1 for s in self.inspection_order if s in required)
+            idx = min(n_found, len(self._REQUIRED_STEP_REWARDS) - 1)
+            return self._REQUIRED_STEP_REWARDS[idx]
+
+        return -0.03       # irrelevant source
 
     def _inspect_feedback(self, source: str, required: list[str], reward: float) -> str:
         label = {"logs": "training logs", "config": "hyperparameter config", "gradients": "gradient statistics"}[source]
-        if source in self.inspected:
+        if source in self.inspection_order:
             return f"You already examined the {label}. No new information gained."
-        if reward > 0:
-            return f"You examined the {label}. This looks relevant."
-        return f"You examined the {label}. This may not be relevant to the failure."
+        if source in required:
+            remaining = len(set(required) - set(self.inspection_order) - {source})
+            msg = f"You examined the {label}. Relevant clue found (+{reward:.2f})."
+            if remaining > 0:
+                msg += f" {remaining} required source(s) still unexamined."
+            return msg
+        return f"You examined the {label}. This source is not required for this failure mode."
 
     def _grade(self, action: WhyDidItFailAction) -> tuple[float, str]:
         """Delegate to the unified grade() function and return (reward, feedback)."""
         assert self.scenario is not None
-        diagnosis    = (action.diagnosis or "").strip().lower()
+        diagnosis     = (action.diagnosis or "").strip().lower()
         suggested_fix = (action.suggested_fix or "").strip().lower() or None
-        difficulty   = self.scenario["difficulty"]
+        difficulty    = self.scenario["difficulty"]
 
         reward = grade(
             diagnosis=diagnosis,
             suggested_fix=suggested_fix,
             scenario=self.scenario,
             steps_taken=self._state.step_count,
-            inspected=self.inspected,
+            inspection_order=self.inspection_order,
             difficulty=difficulty,
         )
 

server/graders.py

@@ -75,21 +75,20 @@ def _diagnosis_score(diagnosis: str, scenario: dict) -> float:
     return max(0.0, min(0.7, score))
 
 
-def _evidence_score(inspected: set[str], required: set[str]) -> float:
+def _evidence_score(inspection_order: list[str], required: set[str]) -> float:
     """
-    +0.05 per required source the agent inspected  (max +0.15 for 3 sources)
-    −0.05 per irrelevant source the agent wasted a step on
-    Clamped to [−0.10, +0.15].
+    +0.08 per required source inspected  (max +0.24 for 3 sources)
+    −0.06 per required source NOT inspected at submit time
+    −0.02 per irrelevant source inspected
+    Clamped to [−0.15, +0.25].
     """
-    relevant   = len(inspected & required)
-    irrelevant = len(inspected - required)
-    score = (relevant * 0.06) - (irrelevant * 0.03)
+    inspected_set = set(inspection_order)
+    relevant   = inspected_set & required
+    missing    = required - inspected_set
+    irrelevant = inspected_set - required
 
-    # small bonus if agent explored more than minimum but not excessively
-    if len(inspected) > len(required):
-        score += 0.02
-
-    return max(-0.10, min(0.15, score))
+    score = (len(relevant) * 0.08) - (len(missing) * 0.06) - (len(irrelevant) * 0.02)
+    return max(-0.15, min(0.25, score))
 
 
 def _efficiency_score(steps_taken: int, min_steps: int) -> float:
@@ -140,7 +139,7 @@ def grade(
     suggested_fix: str | None = None,
     scenario: dict | None = None,
     steps_taken: int = 0,
-    inspected: set[str] | None = None,
+    inspection_order: list[str] | None = None,
     difficulty: str = "easy",   # kept for API compat — not used in scoring logic
 ) -> float:
     """
@@ -152,13 +151,13 @@ def grade(
     Max achievable without fix:  0.70 + 0.15 + 0.15       = 1.00
     Max achievable with fix:     0.70 + 0.15 + 0.15 + 0.15 = 1.00  (capped)
     """
-    scenario  = scenario or {}
-    inspected = inspected or set()
-    required  = set(scenario.get("required_sources", ["logs"]))
-    min_steps = len(required) + 1   # inspect all required sources + submit
+    scenario         = scenario or {}
+    inspection_order = inspection_order or []
+    required         = set(scenario.get("required_sources", ["logs"]))
+    min_steps        = len(required) + 1   # inspect all required sources + submit
 
     d_score = _diagnosis_score(diagnosis, scenario)
-    e_score = _evidence_score(inspected, required)
+    e_score = _evidence_score(inspection_order, required)
     f_score = _efficiency_score(steps_taken, min_steps)
     b_score = _fix_bonus(suggested_fix, scenario)