Upload folder using huggingface_hub

graders.py

@@ -1,25 +1,15 @@
-"""
-Backward-compatible grader re-exports.
-
-The canonical graders now live in server/graders.py.
-This file re-exports them so existing imports still work.
-"""
-
-from server.graders import grade_easy, grade_medium, grade_hard
-
-# Legacy class-based aliases (kept for any old references)
-class Toolforge_grader:
-    def grader(self):
-        return grade_easy()
-
-class EasyGrader(Toolforge_grader):
-    def grade(self, *args, **kwargs):
-        return grade_easy(*args, **kwargs)
-
-class MediumGrader(Toolforge_grader):
-    def grade(self, *args, **kwargs):
-        return grade_medium(*args, **kwargs)
-
-class HardGrader(Toolforge_grader):
-    def grade(self, *args, **kwargs):
-        return grade_hard(*args, **kwargs)
+"""
+Backward-compatible grader re-exports.
+
+The canonical graders now live in server/graders.py.
+This file re-exports them so existing imports still work.
+"""
+
+from server.graders import grade_easy, grade_medium, grade_hard, EpisodeGrader
+
+__all__ = [
+    "grade_easy",
+    "grade_medium",
+    "grade_hard",
+    "EpisodeGrader",
+]

models.py

@@ -135,6 +135,42 @@ class ToolforgeObservation(Observation):
     )
 
 
+class EpisodeGradingState(BaseModel):
+    """Aggregate episode-level signals consumed by the grader.
+    
+    Accumulated incrementally during the episode by the environment.
+    Read by the grader at episode end to compute a normalized score.
+    """
+    # Total steps taken in this episode
+    episode_steps: int = 0
+    # Steps where structural validation failed
+    validation_failures: int = 0
+    # Steps where harmful tool calls were detected
+    harmful_plan_count: int = 0
+    # Steps with full slot completion AND passed validation
+    correct_plan_count: int = 0
+    # Steps where efficiency was computed (slot_ratio == 1.0)
+    fully_correct_efficiency_opportunities: int = 0
+    # Sum of efficiency scores across fully-correct steps
+    sum_efficiency_score: float = 0.0
+    # Times the agent attempted to create a macro
+    macro_creation_attempts: int = 0
+    # Times macro creation was approved by the environment
+    macro_creation_approved: int = 0
+    # Approved AND plan was semantically valid
+    macro_creation_correct: int = 0
+    # Sum of macro creation bonuses awarded
+    macro_creation_bonus_total: float = 0.0
+    # Steps where any accepted macro was used in the plan
+    macro_usage_attempts: int = 0
+    # Macro used AND slot_ratio >= threshold
+    macro_usage_correct: int = 0
+    # Times macro creation was rejected
+    macro_rejected_count: int = 0
+    # Total tasks completed by episode end
+    final_completed_tasks: int = 0
+
+
 class ToolForgeState(State):
     """
     The internal State class for the ToolForge environment.
@@ -177,6 +213,9 @@ class ToolForgeState(State):
     # Macro name -> ordered atomic tool names it represents
     macro_definitions: Dict[str, List[str]] = Field(default_factory=dict)
 
+    # Episode-level grading accumulator (reset each episode)
+    grading: EpisodeGradingState = Field(default_factory=EpisodeGradingState)
+
 
 class PlanAccuracyResult(BaseModel):
     """Output of the Stage-3 plan accuracy calculator."""
@@ -288,6 +327,14 @@ class PipelineResult(BaseModel):
     # Human-readable one-line summary
     summary: str
 
+    # --- Grader-facing step facts (set by pipeline, consumed by environment) ---
+    step_slot_ratio: Optional[float] = None
+    step_task_complete: Optional[bool] = None
+    step_harmful: Optional[bool] = None
+    step_macro_creation_bonus: Optional[float] = None
+    step_macro_usage_bonus: Optional[float] = None
+    step_efficiency_score: Optional[float] = None
+
 
 
 

openenv.yaml

@@ -1,18 +1,18 @@
-spec_version: 1
-name: toolforge_env
-type: space
-runtime: fastapi
-app: server.app:app
-port: 8000
-tasks:
-  - id: easy
-    description: "Repetitive deployment, scaling, and rollback drills with highly repeating tool-call patterns."
-    grader: server.graders.grade_easy
-
-  - id: medium
-    description: "Goal-driven incident response and traffic-readiness scenarios requiring multi-step reasoning."
-    grader: server.graders.grade_medium
-
-  - id: hard
-    description: "Multi-phase infrastructure migration and zero-trust security projects with complex dependencies."
-    grader: server.graders.grade_hard
+spec_version: 1
+name: toolforge_env
+type: space
+runtime: fastapi
+app: server.app:app
+port: 8000
+tasks:
+  - id: easy
+    description: "Repetitive deployment, scaling, and rollback drills with highly repeating tool-call patterns."
+    grader: server.graders.grade_easy
+
+  - id: medium
+    description: "Goal-driven incident response and traffic-readiness scenarios requiring multi-step reasoning."
+    grader: server.graders.grade_medium
+
+  - id: hard
+    description: "Multi-phase infrastructure migration and zero-trust security projects with complex dependencies."
+    grader: server.graders.grade_hard

server/evaluation_pipeline.py

@@ -104,6 +104,12 @@ def run_evaluation_pipeline(
             reward=VALIDATION_PENALTY,
             passed_validation=False,
             summary=f"Validation failed: {validation.reason}",
+            step_slot_ratio=0.0,
+            step_task_complete=False,
+            step_harmful=False,
+            step_macro_creation_bonus=0.0,
+            step_macro_usage_bonus=0.0,
+            step_efficiency_score=0.0,
         )
 
     # 2. run_slot_judgment(...)
@@ -133,6 +139,12 @@ def run_evaluation_pipeline(
             reward=FINAL_REWARD_MIN,
             passed_validation=True,
             summary="Harmful tool call detected.",
+            step_slot_ratio=0.0,
+            step_task_complete=False,
+            step_harmful=True,
+            step_macro_creation_bonus=0.0,
+            step_macro_usage_bonus=0.0,
+            step_efficiency_score=0.0,
         )
 
     # 3. Compute step reward (slot score + macro bonuses + efficiency)
@@ -183,4 +195,10 @@ def run_evaluation_pipeline(
         reward=final_reward,
         passed_validation=True,
         summary="Plan validated, slots judged, final reward applied.",
+        step_slot_ratio=slot_ratio,
+        step_task_complete=slot_judgment.task_complete,
+        step_harmful=False,
+        step_macro_creation_bonus=macro_creation,
+        step_macro_usage_bonus=macro_usage,
+        step_efficiency_score=efficiency_score,
     )

server/graders.py

@@ -1,32 +1,116 @@
-"""
-Grader functions for the ToolForge environment.
-
-Each grader receives the episode trajectory and returns a score in [0, 1].
-These are referenced by openenv.yaml as 'server.graders.grade_easy' etc.
-"""
-
-import random
-
-
-def _base_grade(*args, **kwargs) -> float:
-    """Placeholder grader that returns a random score.
-    
-    Will be replaced with real grading logic that inspects
-    the episode trajectory once the external grader spec is finalised.
-    """
-    return random.uniform(0.01, 0.99)
-
-
-def grade_easy(*args, **kwargs) -> float:
-    """Grade an easy-tier episode."""
-    return _base_grade(*args, **kwargs)
-
-
-def grade_medium(*args, **kwargs) -> float:
-    """Grade a medium-tier episode."""
-    return _base_grade(*args, **kwargs)
-
-
-def grade_hard(*args, **kwargs) -> float:
-    """Grade a hard-tier episode."""
-    return _base_grade(*args, **kwargs)
+"""
+Grader functions for the ToolForge environment.
+
+Each grader receives the complete environment state object and returns
+a benchmark score in [0.01, 0.99]. All three difficulty levels use the
+same EpisodeGrader class — difficulty differences are reflected by the
+episode data itself, not by separate grading logic.
+
+Referenced by openenv.yaml as 'server.graders.grade_easy' etc.
+"""
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class EpisodeGrader:
+    """Reads the episode grading state and computes a normalized [0.01, 0.99] score.
+
+    Sub-scores:
+        accuracy            — correct_plan_count / episode_steps
+        token_optimization  — avg efficiency over fully-correct opportunities
+        macro_creation      — macro_creation_correct / macro_creation_attempts
+        macro_usage         — macro_usage_correct / macro_usage_attempts
+
+    Weights (accuracy > token_opt > macro_creation > macro_usage):
+        0.40 / 0.30 / 0.20 / 0.10
+
+    Final score clamped to [0.01, 0.99].
+    """
+
+    WEIGHTS = {
+        "accuracy": 0.40,
+        "token_optimization": 0.30,
+        "macro_creation": 0.20,
+        "macro_usage": 0.10,
+    }
+
+    def grade(self, state: Any) -> float:
+        """Compute normalized episode score from the environment state.
+
+        Args:
+            state: The ToolForgeState object for the completed episode.
+
+        Returns:
+            Score clamped to [0.01, 0.99].
+        """
+        # Extract the grading accumulator from state
+        g = getattr(state, "grading", None)
+        if g is None:
+            logger.warning("EpisodeGrader: no grading state found, returning 0.01")
+            return 0.01
+
+        # --- Sub-score: accuracy ---
+        if g.episode_steps > 0:
+            accuracy = g.correct_plan_count / g.episode_steps
+        else:
+            accuracy = 0.0
+
+        # --- Sub-score: token optimization ---
+        if g.fully_correct_efficiency_opportunities > 0:
+            token_opt = g.sum_efficiency_score / g.fully_correct_efficiency_opportunities
+        else:
+            token_opt = 0.0
+
+        # --- Sub-score: macro creation ---
+        if g.macro_creation_attempts > 0:
+            macro_create = g.macro_creation_correct / g.macro_creation_attempts
+        else:
+            macro_create = 0.0
+
+        # --- Sub-score: macro usage ---
+        if g.macro_usage_attempts > 0:
+            macro_use = g.macro_usage_correct / g.macro_usage_attempts
+        else:
+            macro_use = 0.0
+
+        raw = (
+            self.WEIGHTS["accuracy"] * accuracy
+            + self.WEIGHTS["token_optimization"] * token_opt
+            + self.WEIGHTS["macro_creation"] * macro_create
+            + self.WEIGHTS["macro_usage"] * macro_use
+        )
+
+        final = max(0.01, min(0.99, raw))
+
+        logger.info(
+            "EpisodeGrader | accuracy=%.3f token_opt=%.3f macro_create=%.3f macro_use=%.3f | raw=%.4f final=%.4f",
+            accuracy, token_opt, macro_create, macro_use, raw, final,
+        )
+
+        return final
+
+
+# Shared grader instance
+_grader = EpisodeGrader()
+
+
+def grade_easy(*args, **kwargs) -> float:
+    """Grade an easy-tier episode."""
+    # The framework passes the state as the first positional argument
+    state = args[0] if args else kwargs.get("state")
+    return _grader.grade(state)
+
+
+def grade_medium(*args, **kwargs) -> float:
+    """Grade a medium-tier episode."""
+    state = args[0] if args else kwargs.get("state")
+    return _grader.grade(state)
+
+
+def grade_hard(*args, **kwargs) -> float:
+    """Grade a hard-tier episode."""
+    state = args[0] if args else kwargs.get("state")
+    return _grader.grade(state)

server/inputs/simulated/task_selector.py

@@ -1,29 +1,29 @@
-from typing import Dict, Any
-
-from server.inputs.simulated.tasks import TASKS
-
-
-TASK_ID_MAP = {
-    "easy": "easy-deployment-sprints",
-    "medium": "medium-traffic-readiness",
-    "hard": "hard-project-legacy-migration",
-}
-
-
-class TaskSelector:
-    def __init__(self):
-        pass
-
-    def next_task_list(self, task_id: str) -> Dict:
-        # Prefer direct internal IDs; if not provided, resolve deterministic aliases.
-        resolved_task_id = task_id
-        direct_match = next((item for item in TASKS if item.get("task_id") == resolved_task_id), None)
-        if direct_match is not None:
-            return direct_match
-
-        resolved_task_id = TASK_ID_MAP.get(task_id, task_id)
-
-        match = next((item for item in TASKS if item.get("task_id") == resolved_task_id), None)
-        if match is None:
-            raise ValueError(f"Unknown task_id: {task_id} (resolved: {resolved_task_id})")
+from typing import Dict, Any
+
+from server.inputs.simulated.tasks import TASKS
+
+
+TASK_ID_MAP = {
+    "easy": "easy-deployment-sprints",
+    "medium": "medium-traffic-readiness",
+    "hard": "hard-project-legacy-migration",
+}
+
+
+class TaskSelector:
+    def __init__(self):
+        pass
+
+    def next_task_list(self, task_id: str) -> Dict:
+        # Prefer direct internal IDs; if not provided, resolve deterministic aliases.
+        resolved_task_id = task_id
+        direct_match = next((item for item in TASKS if item.get("task_id") == resolved_task_id), None)
+        if direct_match is not None:
+            return direct_match
+
+        resolved_task_id = TASK_ID_MAP.get(task_id, task_id)
+
+        match = next((item for item in TASKS if item.get("task_id") == resolved_task_id), None)
+        if match is None:
+            raise ValueError(f"Unknown task_id: {task_id} (resolved: {resolved_task_id})")
         return match

server/toolforge_env_environment.py

@@ -35,6 +35,7 @@ try:
         ToolforgeObservation,
         ToolForgeState,
         Task,
+        EpisodeGradingState,
     )
 except ImportError:
     from models import (
@@ -44,6 +45,7 @@ except ImportError:
         ToolforgeObservation,
         ToolForgeState,
         Task,
+        EpisodeGradingState,
     )
 
 try:
@@ -204,6 +206,9 @@ class ToolforgeEnvironment(Environment):
         self._state.macro_definitions = prev_macro_defs
         self._last_approval = None
 
+        # Fresh episode-level grading accumulator
+        self._state.grading = EpisodeGradingState()
+
 
         obs = self._create_default_observation()
         obs.done = False
@@ -239,6 +244,10 @@ class ToolforgeEnvironment(Environment):
             self._state.step_count += 1
             self._last_approval = False
 
+            # Track malformed actions in grading state
+            self._state.grading.episode_steps += 1
+            self._state.grading.validation_failures += 1
+
             progression = "advanced_to_next_task_malformed_action"
             if self._state.step_count >= self.MAX_EPISODE_STEPS:
                 self._state.done = True
@@ -312,6 +321,9 @@ class ToolforgeEnvironment(Environment):
         if macro_result["decision"] == "approved" and action.macro_proposal is not None:
             self._state.available_tools.append(action.macro_proposal)
 
+        # Accumulate grading state from this step
+        self._update_grading_state(pipeline_result, macro_result, action)
+
         # Fetch the scalar reward from the evaluation pipeline
         reward = float(pipeline_result.reward)
         print(self._state.current_task.prompt, reward, progression, macro_result)
@@ -350,6 +362,56 @@ class ToolforgeEnvironment(Environment):
         logger.debug("Next task prompt: %s", next_task.prompt)
         return True
 
+    def _update_grading_state(
+        self,
+        pipeline_result,
+        macro_result: Dict[str, Any],
+        action: ToolforgeAction,
+    ) -> None:
+        """Accumulate episode-level grading counters from a single step."""
+        g = self._state.grading
+        g.episode_steps += 1
+
+        # Validation failures
+        if not pipeline_result.passed_validation:
+            g.validation_failures += 1
+
+        # Harmful calls
+        if pipeline_result.step_harmful:
+            g.harmful_plan_count += 1
+
+        # Correct plans (full slot fill + valid)
+        if pipeline_result.step_task_complete and pipeline_result.passed_validation:
+            g.correct_plan_count += 1
+
+        # Efficiency tracking (only when slot_ratio == 1.0)
+        sr = pipeline_result.step_slot_ratio
+        if sr is not None and sr >= 1.0:
+            g.fully_correct_efficiency_opportunities += 1
+            g.sum_efficiency_score += (pipeline_result.step_efficiency_score or 0.0)
+
+        # Macro creation tracking
+        if macro_result["attempted"]:
+            g.macro_creation_attempts += 1
+            if macro_result["decision"] == "approved":
+                g.macro_creation_approved += 1
+                if pipeline_result.passed_validation:
+                    g.macro_creation_correct += 1
+                g.macro_creation_bonus_total += (pipeline_result.step_macro_creation_bonus or 0.0)
+            elif macro_result["decision"] == "rejected":
+                g.macro_rejected_count += 1
+
+        # Macro usage tracking
+        macro_names = {m.name for m in self._state.accepted_macros}
+        if any(c.tool_name in macro_names for c in action.plan):
+            g.macro_usage_attempts += 1
+            if sr is not None and sr >= 0.65:
+                g.macro_usage_correct += 1
+
+        # Keep completed task count current
+        g.final_completed_tasks = len(self._state.completed_tasks)
+
+
     def _analyze_plan(self, plan: List[ToolCall]) -> Dict[str, Any]:
         """Compute deterministic call accounting from server-side tool registry."""