Rewrite graders: grade(episode)->float, no external deps, difficulty progression

server.py

@@ -546,13 +546,13 @@ async def grade_episode(req: GradeRequest):
         raise HTTPException(status_code=400, detail=f"Unknown task_id '{task_id}'")
 
     try:
-        # Use per-task grader module (tasks/{task_id}/grader.py)
-        mod = importlib.import_module(f"tasks.{task_id}.grader")
-        result = mod.grade_episode(req.episode or {})
+        mod   = importlib.import_module(f"tasks.{task_id}.grader")
+        score = mod.grade(req.episode or {})   # grade(episode) -> float
+        score = float(score)
         return {
-            "task_id": task_id,
-            "score":       result["score"],
-            "passed":      result["passed"],
+            "task_id":     task_id,
+            "score":       round(max(0.005, min(0.995, score)), 4),
+            "passed":      True,
             "score_range": [0.0, 1.0],
             "grader":      f"tasks.{task_id}.grader.grade",
         }

tasks/easy/grader.py

@@ -1,39 +1,47 @@
-"""
-tasks/easy/grader.py
-====================
-Grader for the easy content-moderation task.
-
-Exposes a module-level ``grade(output, expected)`` function that the
-OpenEnv submission platform calls to verify this task has a working grader.
-
-Also delegates to EasyGrader for full episode scoring.
-"""
-
+"""tasks/easy/grader.py — OpenEnv-compliant grader for the easy task."""
 from __future__ import annotations
 
-from tasks.graders import EasyGrader
-
-_grader = EasyGrader()
-
-
-def grade(output: str, expected: str) -> float:
-    out = str(output).strip().lower()
-    exp = str(expected).strip().lower()
-
-    if out == exp:
-        score = 1.0
-    elif (out, exp) in {
-        ("flag", "remove"), ("remove", "flag"),
-        ("flag", "allow"), ("allow", "flag")
-    }:
-        score = 0.5
-    else:
-        score = 0.0
-
-    return float(score)
-
-
-def grade_episode(episode: dict) -> dict:
-    """Grade a full episode dict and return score + passed."""
-    score = _grader.grade(episode)
-    return {"score": score, "passed": True}
+_CREDIT = {
+    ("allow",  "allow"):  1.0,
+    ("flag",   "flag"):   1.0,
+    ("remove", "remove"): 1.0,
+    ("flag",   "remove"): 0.5,
+    ("remove", "flag"):   0.5,
+    ("flag",   "allow"):  0.5,
+    ("allow",  "flag"):   0.5,
+    ("allow",  "remove"): 0.0,
+    ("remove", "allow"):  0.0,
+}
+
+
+def grade(episode: dict) -> float:
+    """
+    OpenEnv grader — easy difficulty.
+    Returns a float strictly in (0.0, 1.0).
+
+    Weights: decision 50% | evidence 25% | reasoning 15% | efficiency 10%
+    """
+    decision     = (episode.get("final_decision") or
+                    episode.get("decision") or "none").lower().strip()
+    ground_truth = (episode.get("ground_truth") or
+                    episode.get("correct_decision") or "none").lower().strip()
+
+    decision_score = _CREDIT.get((decision, ground_truth), 0.0)
+
+    all_src        = episode.get("all_source_ids") or []
+    checked        = episode.get("sources_checked") or []
+    evidence_score = min(1.0, len(set(checked)) / len(all_src)) if all_src else 1.0
+
+    all_clm          = episode.get("all_claim_ids") or []
+    verified         = episode.get("claims_verified") or []
+    reasoning_score  = min(1.0, len(set(verified)) / len(all_clm)) if all_clm else 1.0
+
+    steps            = int(episode.get("step_count") or 0)
+    efficiency_score = 1.0 if steps <= 8 else max(0.0, 1.0 - 0.1 * (steps - 8))
+
+    raw = (0.50 * decision_score +
+           0.25 * evidence_score +
+           0.15 * reasoning_score +
+           0.10 * efficiency_score)
+
+    return round(max(0.005, min(0.995, raw)), 4)

tasks/hard/grader.py

@@ -1,37 +1,58 @@
-"""
-tasks/hard/grader.py
-====================
-Grader for the hard content-moderation task.
-
-Exposes a module-level ``grade(output, expected)`` function that the
-OpenEnv submission platform calls to verify this task has a working grader.
-"""
-
+"""tasks/hard/grader.py — OpenEnv-compliant grader for the hard task."""
 from __future__ import annotations
 
-from tasks.graders import HardGrader
-
-_grader = HardGrader()
-
-
-def grade(output: str, expected: str) -> float:
-    out = str(output).strip().lower()
-    exp = str(expected).strip().lower()
-
-    if out == exp:
-        score = 1.0
-    elif (out, exp) in {
-        ("flag", "remove"), ("remove", "flag"),
-        ("flag", "allow"), ("allow", "flag")
-    }:
-        score = 0.5
+_CREDIT = {
+    ("allow",  "allow"):  1.0,
+    ("flag",   "flag"):   1.0,
+    ("remove", "remove"): 1.0,
+    ("flag",   "remove"): 0.5,
+    ("remove", "flag"):   0.5,
+    ("flag",   "allow"):  0.5,
+    ("allow",  "flag"):   0.5,
+    ("allow",  "remove"): 0.25,  # hard: never 0 for non-trivially wrong
+    ("remove", "allow"):  0.25,
+}
+
+
+def grade(episode: dict) -> float:
+    """
+    OpenEnv grader — hard difficulty.
+    Returns a float strictly in (0.0, 1.0).
+
+    Weights: decision 30% | evidence 30% | reasoning 30% | efficiency 10%
+    All claims must be verified for full reasoning score.
+    """
+    decision     = (episode.get("final_decision") or
+                    episode.get("decision") or "none").lower().strip()
+    ground_truth = (episode.get("ground_truth") or
+                    episode.get("correct_decision") or "none").lower().strip()
+
+    decision_score = _CREDIT.get((decision, ground_truth), 0.0)
+
+    all_src        = episode.get("all_source_ids") or []
+    checked        = episode.get("sources_checked") or []
+    frac_src       = (len(set(checked)) / len(all_src)) if all_src else 1.0
+    # Hard requires 60% minimum for any credit
+    if frac_src >= 1.0:
+        evidence_score = 1.0
+    elif frac_src >= 0.60:
+        evidence_score = 0.5 + 0.5 * (frac_src - 0.60) / 0.40
     else:
-        score = 0.0
+        evidence_score = frac_src / 0.60 * 0.5
+
+    all_clm         = episode.get("all_claim_ids") or []
+    verified        = episode.get("claims_verified") or []
+    frac_clm        = (len(set(verified)) / len(all_clm)) if all_clm else 1.0
+    # Hard requires ALL claims verified for full score
+    reasoning_score = min(1.0, 0.6 + 0.4 * frac_clm) if frac_clm >= 0.5 else frac_clm * 0.6 / 0.5
 
-    return float(score)
+    steps            = int(episode.get("step_count") or 0)
+    # Hard: efficiency barely penalised — thorough exploration is rewarded
+    efficiency_score = 1.0 if steps <= 18 else max(0.0, 1.0 - 0.05 * (steps - 18))
 
+    raw = (0.30 * decision_score +
+           0.30 * evidence_score +
+           0.30 * reasoning_score +
+           0.10 * efficiency_score)
 
-def grade_episode(episode: dict) -> dict:
-    """Grade a full episode dict and return score + passed."""
-    score = _grader.grade(episode)
-    return {"score": score, "passed": True}
+    return round(max(0.005, min(0.995, raw)), 4)

tasks/medium/grader.py

@@ -1,36 +1,58 @@
-"""
-tasks/medium/grader.py
-======================
-Grader for the medium content-moderation task.
-
-Exposes a module-level ``grade(output, expected)`` function that the
-OpenEnv submission platform calls to verify this task has a working grader.
-"""
-
+"""tasks/medium/grader.py — OpenEnv-compliant grader for the medium task."""
 from __future__ import annotations
 
-from tasks.graders import MediumGrader
-
-_grader = MediumGrader()
-
+_CREDIT = {
+    ("allow",  "allow"):  1.0,
+    ("flag",   "flag"):   1.0,
+    ("remove", "remove"): 1.0,
+    ("flag",   "remove"): 0.5,
+    ("remove", "flag"):   0.5,
+    ("flag",   "allow"):  0.5,
+    ("allow",  "flag"):   0.5,
+    ("allow",  "remove"): 0.0,
+    ("remove", "allow"):  0.0,
+}
+
+
+def grade(episode: dict) -> float:
+    """
+    OpenEnv grader — medium difficulty.
+    Returns a float strictly in (0.0, 1.0).
+
+    Weights: decision 35% | evidence 35% | reasoning 20% | efficiency 10%
+    Higher evidence and reasoning requirements than easy.
+    """
+    decision     = (episode.get("final_decision") or
+                    episode.get("decision") or "none").lower().strip()
+    ground_truth = (episode.get("ground_truth") or
+                    episode.get("correct_decision") or "none").lower().strip()
+
+    decision_score = _CREDIT.get((decision, ground_truth), 0.0)
+
+    all_src        = episode.get("all_source_ids") or []
+    checked        = episode.get("sources_checked") or []
+    frac_src       = (len(set(checked)) / len(all_src)) if all_src else 1.0
+    # Medium requires 75% source coverage for full score
+    if frac_src >= 0.75:
+        evidence_score = 1.0
+    elif frac_src >= 0.50:
+        evidence_score = 0.6 + 0.4 * (frac_src - 0.50) / 0.25
+    elif frac_src >= 0.33:
+        evidence_score = 0.3 + 0.3 * (frac_src - 0.33) / 0.17
+    else:
+        evidence_score = frac_src / 0.33 * 0.3
 
-def grade(output: str, expected: str) -> float:
-    out = str(output).strip().lower()
-    exp = str(expected).strip().lower()
+    all_clm         = episode.get("all_claim_ids") or []
+    verified        = episode.get("claims_verified") or []
+    frac_clm        = (len(set(verified)) / len(all_clm)) if all_clm else 1.0
+    reasoning_score = min(1.0, frac_clm / 0.75) if frac_clm < 0.75 else 1.0
 
-    if out == exp:
-        score = 1.0
-    elif (out, exp) in {
-        ("flag", "remove"), ("remove", "flag"),
-        ("flag", "allow"), ("allow", "flag")
-    }:
-        score = 0.5
-    else:
-        score = 0.0
+    steps            = int(episode.get("step_count") or 0)
+    efficiency_score = 1.0 if steps <= 12 else max(0.0, 1.0 - 0.1 * (steps - 12))
 
-    return float(score)
+    raw = (0.35 * decision_score +
+           0.35 * evidence_score +
+           0.20 * reasoning_score +
+           0.10 * efficiency_score)
 
-def grade_episode(episode: dict) -> dict:
-    """Grade a full episode dict and return score + passed."""
-    score = _grader.grade(episode)
-    return {"score": score, "passed": True}
+    return round(max(0.005, min(0.995, raw)), 4)