Add per-task grader.py/task.py structure; update /tasks and /grade endpoints

openenv.yaml

@@ -151,8 +151,8 @@ tasks:
     n_scenarios: 5
     grader:
       enabled: true
-      class: tasks.graders.EasyGrader
-      method: grade
+      class: tasks.easy.grader.grade
+      fn: tasks.easy.grader.grade
       score_range: [0.0, 1.0]
     weights: { decision: 0.50, evidence: 0.25, reasoning: 0.15, efficiency: 0.10 }
     ideal_max_steps: 8
@@ -168,8 +168,8 @@ tasks:
     n_scenarios: 5
     grader:
       enabled: true
-      class: tasks.graders.MediumGrader
-      method: grade
+      class: tasks.medium.grader.grade
+      fn: tasks.medium.grader.grade
       score_range: [0.0, 1.0]
     weights: { decision: 0.35, evidence: 0.35, reasoning: 0.20, efficiency: 0.10 }
     ideal_max_steps: 12
@@ -185,8 +185,8 @@ tasks:
     n_scenarios: 5
     grader:
       enabled: true
-      class: tasks.graders.HardGrader
-      method: grade
+      class: tasks.hard.grader.grade
+      fn: tasks.hard.grader.grade
       score_range: [0.0, 1.0]
     weights: { decision: 0.30, evidence: 0.30, reasoning: 0.30, efficiency: 0.10 }
     ideal_max_steps: 18

server.py

@@ -312,10 +312,12 @@ async def list_tasks():
                 "scenario_ids": [s["id"] for s in task.get_all_scenarios()],
                 "grader": {
                     "enabled": True,
-                    "class": f"tasks.graders.{grader.__class__.__name__}",
-                    "method": "grade",
-                    "score_range": [0.0, 1.0]
+                    "class": f"tasks.{difficulty}.grader.grade",
+                    "fn":    f"tasks.{difficulty}.grader.grade",
+                    "score_range": [0.0, 1.0],
                 },
+                "grader_fn":     f"tasks.{difficulty}.grader.grade",
+                "score_range":   [0.0, 1.0],
                 "grader_weights": {
                     "decision":   grader.W_DECISION,
                     "evidence":   grader.W_EVIDENCE,
@@ -539,23 +541,26 @@ class GradeRequest(BaseModel):
 async def grade_episode(req: GradeRequest):
     """
     Grade a completed episode for a specific task.
-    This endpoint allows the OpenEnv validator to test graders.
+    Accepts an optional episode dict; if omitted, probes the grader with empty input.
+    Returns {"score", "passed", "score_range", "grader"}.
     """
-    from tasks.graders import GraderFactory
-    
+    import importlib
+
+    task_id = req.task_id.lower().strip()
+    if task_id not in ("easy", "medium", "hard"):
+        raise HTTPException(status_code=400, detail=f"Unknown task_id '{task_id}'")
+
     try:
-        grader = GraderFactory.get(req.task_id)
-        score = grader.grade(req.episode or {})
-        
+        # Use per-task grader module (tasks/{task_id}/grader.py)
+        mod = importlib.import_module(f"tasks.{task_id}.grader")
+        result = mod.grade_episode(req.episode or {})
         return {
-            "task_id": req.task_id,
-            "score": score,
-            "passed": True,       # grader ran successfully and returned a valid score
+            "task_id": task_id,
+            "score":       result["score"],
+            "passed":      result["passed"],
             "score_range": [0.0, 1.0],
-            "grader": f"tasks.graders.{grader.__class__.__name__}"
+            "grader":      f"tasks.{task_id}.grader.grade",
         }
-    except ValueError as e:
-        raise HTTPException(status_code=400, detail=str(e))
     except Exception as e:
         logger.error("Grading failed: %s", traceback.format_exc())
         raise HTTPException(status_code=500, detail=f"Grading failed: {str(e)}")

tasks/easy/grader.py

@@ -0,0 +1,49 @@
+"""
+tasks/easy/grader.py
+====================
+Grader for the easy content-moderation task.
+
+Exposes a module-level ``grade(output, expected)`` function that the
+OpenEnv submission platform calls to verify this task has a working grader.
+
+Also delegates to EasyGrader for full episode scoring.
+"""
+
+from __future__ import annotations
+
+from tasks.graders import EasyGrader
+
+_grader = EasyGrader()
+
+
+def grade(output: str, expected: str) -> dict:
+    """
+    Grade a single moderation decision against the expected label.
+
+    Parameters
+    ----------
+    output   : agent's decision  ("allow" | "flag" | "remove")
+    expected : ground-truth      ("allow" | "flag" | "remove")
+
+    Returns
+    -------
+    {"score": float, "passed": bool}
+    """
+    out = str(output).strip().lower()
+    exp = str(expected).strip().lower()
+
+    if out == exp:
+        score = 1.0
+    elif (out, exp) in {("flag", "remove"), ("remove", "flag"),
+                        ("flag", "allow"), ("allow", "flag")}:
+        score = 0.5
+    else:
+        score = 0.0
+
+    return {"score": score, "passed": score > 0.0}
+
+
+def grade_episode(episode: dict) -> dict:
+    """Grade a full episode dict and return score + passed."""
+    score = _grader.grade(episode)
+    return {"score": score, "passed": True}

tasks/easy/task.py

@@ -0,0 +1,19 @@
+"""
+tasks/easy/task.py
+==================
+Task definition for the easy difficulty tier.
+
+Clear-cut misinformation: overwhelming contradicting evidence from
+authoritative sources (WHO, CDC, NASA). Correct decision is unambiguous.
+"""
+
+TASK_ID = "easy"
+DESCRIPTION = (
+    "Content moderation — easy difficulty. "
+    "Agent must verify claims against authoritative sources and make a "
+    "moderation decision (allow / flag / remove). "
+    "Correct decisions are clear-cut with overwhelming evidence."
+)
+EXPECTED_DECISION_EXAMPLES = ["remove", "flag", "allow"]
+SCORE_RANGE = [0.0, 1.0]
+N_SCENARIOS = 5

tasks/hard/grader.py

@@ -0,0 +1,47 @@
+"""
+tasks/hard/grader.py
+====================
+Grader for the hard content-moderation task.
+
+Exposes a module-level ``grade(output, expected)`` function that the
+OpenEnv submission platform calls to verify this task has a working grader.
+"""
+
+from __future__ import annotations
+
+from tasks.graders import HardGrader
+
+_grader = HardGrader()
+
+
+def grade(output: str, expected: str) -> dict:
+    """
+    Grade a single moderation decision against the expected label.
+
+    Parameters
+    ----------
+    output   : agent's decision  ("allow" | "flag" | "remove")
+    expected : ground-truth      ("allow" | "flag" | "remove")
+
+    Returns
+    -------
+    {"score": float, "passed": bool}
+    """
+    out = str(output).strip().lower()
+    exp = str(expected).strip().lower()
+
+    if out == exp:
+        score = 1.0
+    elif (out, exp) in {("flag", "remove"), ("remove", "flag"),
+                        ("flag", "allow"), ("allow", "flag")}:
+        score = 0.5
+    else:
+        score = 0.0
+
+    return {"score": score, "passed": score > 0.0}
+
+
+def grade_episode(episode: dict) -> dict:
+    """Grade a full episode dict and return score + passed."""
+    score = _grader.grade(episode)
+    return {"score": score, "passed": True}

tasks/hard/task.py

@@ -0,0 +1,19 @@
+"""
+tasks/hard/task.py
+==================
+Task definition for the hard difficulty tier.
+
+Conflicting evidence from credible sources; genuinely contested claims.
+Requires nuanced multi-claim reasoning and tolerance for ambiguity.
+"""
+
+TASK_ID = "hard"
+DESCRIPTION = (
+    "Content moderation — hard difficulty. "
+    "Agent must reason over conflicting evidence from credible sources and "
+    "make a moderation decision (allow / flag / remove). "
+    "Ground truth is ambiguous — all scoring dimensions count equally."
+)
+EXPECTED_DECISION_EXAMPLES = ["flag"]
+SCORE_RANGE = [0.0, 1.0]
+N_SCENARIOS = 5

tasks/medium/grader.py

@@ -0,0 +1,47 @@
+"""
+tasks/medium/grader.py
+======================
+Grader for the medium content-moderation task.
+
+Exposes a module-level ``grade(output, expected)`` function that the
+OpenEnv submission platform calls to verify this task has a working grader.
+"""
+
+from __future__ import annotations
+
+from tasks.graders import MediumGrader
+
+_grader = MediumGrader()
+
+
+def grade(output: str, expected: str) -> dict:
+    """
+    Grade a single moderation decision against the expected label.
+
+    Parameters
+    ----------
+    output   : agent's decision  ("allow" | "flag" | "remove")
+    expected : ground-truth      ("allow" | "flag" | "remove")
+
+    Returns
+    -------
+    {"score": float, "passed": bool}
+    """
+    out = str(output).strip().lower()
+    exp = str(expected).strip().lower()
+
+    if out == exp:
+        score = 1.0
+    elif (out, exp) in {("flag", "remove"), ("remove", "flag"),
+                        ("flag", "allow"), ("allow", "flag")}:
+        score = 0.5
+    else:
+        score = 0.0
+
+    return {"score": score, "passed": score > 0.0}
+
+
+def grade_episode(episode: dict) -> dict:
+    """Grade a full episode dict and return score + passed."""
+    score = _grader.grade(episode)
+    return {"score": score, "passed": True}

tasks/medium/task.py

@@ -0,0 +1,19 @@
+"""
+tasks/medium/task.py
+====================
+Task definition for the medium difficulty tier.
+
+Partially true or misleading content with mixed evidence. Requires
+careful claim-by-claim verification and evidence weighing.
+"""
+
+TASK_ID = "medium"
+DESCRIPTION = (
+    "Content moderation — medium difficulty. "
+    "Agent must verify claims, evaluate mixed evidence sources, and make a "
+    "moderation decision (allow / flag / remove). "
+    "Content is partially true or misleading — nuanced reasoning required."
+)
+EXPECTED_DECISION_EXAMPLES = ["flag", "allow"]
+SCORE_RANGE = [0.0, 1.0]
+N_SCENARIOS = 5