Update api/server.py

api/server.py

@@ -19,7 +19,7 @@ from env.models import (
     HealthResponse, TaskInfo, ProgressResponse
 )
 from env.tasks import task_manager, ACTION_SCHEMA
-from env.graders import grade
+from env.graders import grade, grade_db_action, _is_scenario_task, _get_scenario
 
 
 # ─────────────────────────────────────────────
@@ -143,8 +143,8 @@ async def step(action: Action):
         return StepResponse(
             observation = environment._build_observation(),
             reward      = Reward(
-                score     = 0.0,
-                breakdown = {"validation_error": 0.0},
+                score     = 0.001,
+                breakdown = {"validation_error": 0.001},
                 feedback  = f"Malformed action: {str(e)}"
             ),
             done = False,
@@ -183,67 +183,154 @@ async def tasks():
 
 
 # ─────────────────────────────────────────────
-#  6. /grader — POST
+#  6. /grader — POST  (FIXED)
 # ─────────────────────────────────────────────
 
 @app.post("/grader", response_model=GraderResponse, tags=["Grading"])
 async def grader(request: GraderRequest):
     """
-    Grades a completed episode action.
-    For Round 2 submit_report: computes score from DB performance improvement.
-    Returns float score strictly between 0.0 and 1.0 exclusive.
+    Grades an action for a given task_id. STATELESS — does not change episode state.
+
+    Routing:
+      Round 2 scenario IDs (easy_s001, medium_s002, hard_s003):
+        - submit_report   → computes score from current DB performance delta
+        - all other types → grade_db_action() scores action quality vs scenario
+
+      Round 1 task IDs (easy_001, medium_001, hard_001):
+        → grade() → grade_easy/medium/hard() (original Round 1 graders)
+
+    Score is ALWAYS strictly between 0.001 and 0.999.
+    NEVER crashes — all exceptions caught and returned as 0.001.
+
+    FIXES applied vs original:
+      - Round 2 non-terminal actions now route to grade_db_action() instead of
+        grade_easy() which was looking for "fixed_query" in Round 2 payloads
+        and returning 0.001 for every create_index / analyze_indexes / inspect_query
+      - submit_report score now uses db_simulator state from environment directly
+        instead of brittle action_counts dict lookup which could be empty or stale
     """
     try:
         if request.action is None:
             return GraderResponse(
-                score     = 0.0,
+                score     = 0.001,
                 feedback  = "No action provided for grading.",
                 breakdown = {"error": "null_action"}
             )
 
-        # Round 2: submit_report grading uses DB state
-        if request.action.action_type == ActionType.SUBMIT_REPORT:
-            ep_state    = environment.state()
-            perf_history = ep_state.action_counts.get("_perf_history", [0.0])
-            baseline     = ep_state.action_counts.get("_baseline_score", 0.0)
-            best_score   = ep_state.action_counts.get("_best_score", 0.0)
-            current      = perf_history[-1] if perf_history else 0.0
-            max_possible = max(1.0, 100.0 - baseline)
-
-            perf_improvement = (current - baseline) / max_possible
-            step_efficiency  = 1.0 - (ep_state.step_count / max(1, 50))
-            score = round(
-                (perf_improvement * 0.60) + (step_efficiency * 0.20) + 0.10, 4
-            )
-            score = max(0.0, min(1.0, score))
+        task_id     = request.task_id or ""
+        action_type = (
+            request.action.action_type.value
+            if hasattr(request.action.action_type, "value")
+            else str(request.action.action_type)
+        )
 
-            return GraderResponse(
-                score    = score,
-                feedback = (
-                    f"DB performance: {baseline:.1f} → {current:.1f} "
-                    f"(best: {best_score:.1f}). "
-                    f"Steps used: {ep_state.step_count}/50."
-                ),
-                breakdown = {
-                    "perf_improvement": round(perf_improvement, 4),
-                    "step_efficiency":  round(step_efficiency, 4),
-                    "base_score":       0.10,
-                }
-            )
+        # ── ROUND 2: DB ENGINEERING SCENARIO ─────────────────────
+        if _is_scenario_task(task_id):
+
+            # submit_report: use live DB state from environment simulator
+            if action_type == "submit_report":
+                return _grade_submit_report(request, task_id)
+
+            # All other Round 2 actions: stateless scenario-aware grading
+            score, breakdown, feedback = grade_db_action(request.action, task_id)
+            score = max(0.001, min(0.999, score))
+            return GraderResponse(score=score, feedback=feedback, breakdown=breakdown)
 
-        # Round 1 grading
-        score, breakdown, feedback = grade(request.action, request.task_id)
-        score = max(0.0, min(1.0, score))
+        # ── ROUND 1: SQL DEBUGGING TASK ───────────────────────────
+        score, breakdown, feedback = grade(request.action, task_id)
+        score = max(0.001, min(0.999, score))
         return GraderResponse(score=score, feedback=feedback, breakdown=breakdown)
 
     except Exception as e:
         return GraderResponse(
-            score     = 0.0,
+            score     = 0.001,
             feedback  = f"Grader error: {str(e)}",
             breakdown = {"error": str(e)}
         )
 
 
+def _grade_submit_report(request: GraderRequest, task_id: str) -> GraderResponse:
+    """
+    Grade a submit_report action for a Round 2 scenario.
+
+    Score components:
+      60% — performance improvement (baseline → current)
+      20% — step efficiency (fewer steps = higher bonus)
+      10% — base credit for submitting
+      10% — report summary quality
+
+    Falls back gracefully if DB simulator state is unavailable.
+    """
+    try:
+        ep_state = environment.state()
+
+        # Get performance data from environment state
+        # Use action_counts as the store (set by environment.py during steps)
+        ac          = ep_state.action_counts or {}
+        perf_history = ac.get("_perf_history", [])
+        baseline     = float(ac.get("_baseline_score", 0.0))
+        current      = float(perf_history[-1]) if perf_history else baseline
+        steps_used   = ep_state.step_count
+        max_steps    = 50  # Round 2 default
+
+        # If no perf history (called before reset, or env in wrong state):
+        # fall back to scenario-based quality score
+        if not perf_history or baseline == 0.0:
+            scenario = _get_scenario(task_id)
+            if scenario:
+                baseline = float(scenario.get("performance_score_baseline", 0.0))
+                target   = float(scenario.get("target_score", 85.0))
+                # Score based on report quality only
+                summary    = str((request.action.payload or {}).get("summary", ""))
+                base_score = 0.15 + min(len(summary) / 400, 0.25)
+                return GraderResponse(
+                    score    = round(max(0.001, min(0.999, base_score)), 4),
+                    feedback = (
+                        f"Report graded on quality only (episode state unavailable). "
+                        f"Run a full episode via /reset then /step to get performance-based score."
+                    ),
+                    breakdown = {"report_quality": round(base_score, 4), "note": "no_episode_state"}
+                )
+
+        max_possible     = max(1.0, 100.0 - baseline)
+        perf_improvement = max(0.0, (current - baseline) / max_possible)
+        step_efficiency  = max(0.0, 1.0 - (steps_used / max(1, max_steps)))
+        summary          = str((request.action.payload or {}).get("summary", ""))
+        report_quality   = min(len(summary) / 300, 0.10) if summary else 0.0
+
+        raw_score = (
+            perf_improvement * 0.60
+            + step_efficiency * 0.20
+            + 0.10                  # base credit
+            + report_quality        # up to 0.10
+        )
+        score = round(max(0.001, min(0.999, raw_score)), 4)
+
+        return GraderResponse(
+            score    = score,
+            feedback = (
+                f"DB performance: {baseline:.1f} → {current:.1f} "
+                f"(improvement: {perf_improvement*100:.1f}%). "
+                f"Steps used: {steps_used}/{max_steps}. "
+                f"Efficiency: {step_efficiency*100:.1f}%."
+            ),
+            breakdown = {
+                "perf_improvement": round(perf_improvement, 4),
+                "step_efficiency":  round(step_efficiency,  4),
+                "base_credit":      0.10,
+                "report_quality":   round(report_quality,   4),
+            }
+        )
+
+    except Exception as e:
+        # Last resort — don't return an error, return a low but non-zero score
+        return GraderResponse(
+            score     = 0.10,
+            feedback  = f"Submit report scored with fallback (error: {str(e)}).",
+            breakdown = {"fallback": 0.10, "error": str(e)}
+        )
+
+
 # ─────────────────────────────────────────────
 #  7. /baseline — POST
 # ─────────────────────────────────────────────
@@ -280,7 +367,7 @@ async def baseline():
 
 
 # ─────────────────────────────────────────────
-#  8. /progress — GET  (Round 2 NEW)
+#  8. /progress — GET  (Round 2)
 # ─────────────────────────────────────────────
 
 @app.get("/progress", response_model=ProgressResponse, tags=["Training"])
@@ -291,7 +378,7 @@ async def progress():
     Shows improvement from baseline to current score.
     """
     ep_state     = environment.state()
-    ac           = ep_state.action_counts
+    ac           = ep_state.action_counts or {}
     perf_history = ac.get("_perf_history", [])
     milestones   = ac.get("_milestones", [])
     baseline     = ac.get("_baseline_score", 0.0)
@@ -331,4 +418,4 @@ async def root():
         "tasks_count": 30,
         "max_steps":   50,
         "themes":      ["Long-Horizon Planning", "World Modeling", "Self-Improvement", "Wildcard"],
-    }
+    }