Upload folder using huggingface_hub

graders.py

@@ -1,21 +1,25 @@
-import random
-
-
-class Toolforge_grader:
-    def grader(self):
-        return random.uniform(0.01, 0.99)
-
-
-class EasyGrader(Toolforge_grader):
-    def grade(self, *args, **kwargs):
-        return self.grader()
-
-
-class MediumGrader(Toolforge_grader):
-    def grade(self, *args, **kwargs):
-        return self.grader()
-
-
-class HardGrader(Toolforge_grader):
-    def grade(self, *args, **kwargs):
-        return self.grader()
+"""
+Backward-compatible grader re-exports.
+
+The canonical graders now live in server/graders.py.
+This file re-exports them so existing imports still work.
+"""
+
+from server.graders import grade_easy, grade_medium, grade_hard
+
+# Legacy class-based aliases (kept for any old references)
+class Toolforge_grader:
+    def grader(self):
+        return grade_easy()
+
+class EasyGrader(Toolforge_grader):
+    def grade(self, *args, **kwargs):
+        return grade_easy(*args, **kwargs)
+
+class MediumGrader(Toolforge_grader):
+    def grade(self, *args, **kwargs):
+        return grade_medium(*args, **kwargs)
+
+class HardGrader(Toolforge_grader):
+    def grade(self, *args, **kwargs):
+        return grade_hard(*args, **kwargs)

inference.py

@@ -44,7 +44,6 @@ STDOUT FORMAT
 import asyncio
 import json
 import os
-import pprint
 import textwrap
 from typing import List, Optional, Dict, Any
 
@@ -57,13 +56,18 @@ API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")
 
 API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co/v1"
 MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct"
-TASK_NAME = os.getenv("MY_ENV_V4_TASK", "easy-deployment-sprints")
 BENCHMARK = os.getenv("MY_ENV_V4_BENCHMARK", "toolforge_env")
 MAX_STEPS = 8
 TEMPERATURE = 0.7
 MAX_TOKENS = 500
 SUCCESS_SCORE_THRESHOLD = 0.1  # normalized score in [0, 1]
 
+TASKS = [
+    "easy",
+    "medium",
+    "hard",
+]
+
 # Max possible reward: each token contributes 0.1, across all steps
 _MAX_REWARD_PER_STEP = MAX_TOKENS * 0.1
 MAX_TOTAL_REWARD = MAX_STEPS * _MAX_REWARD_PER_STEP
@@ -124,6 +128,18 @@ def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> No
     print(f"[END] success={str(success).lower()} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)
 
 
+def get_task_list() -> List[str]:
+    raw_tasks = os.getenv("MY_ENV_V4_TASKS", "")
+    if raw_tasks.strip():
+        return [task.strip() for task in raw_tasks.split(",") if task.strip()]
+
+    single_task = os.getenv("MY_ENV_V4_TASK", "")
+    if single_task.strip():
+        return [single_task.strip()]
+
+    return TASKS
+
+
 def build_user_prompt(step: int, current_task: Any, available_tools: List[Dict[str, Any]], last_reward: float, history: List[str]) -> str:
     history_block = "\n".join(history[-4:]) if history else "None"
 
@@ -188,14 +204,12 @@ def get_model_action(
         )
         raw = completion.choices[0].message.content
         if raw is None:
-            print(f"[DEBUG] Model returned None content")
             return build_fallback_action(available_tools, current_task)
         raw = raw.strip()
         raw = raw.replace("```json", "").replace("```", "").strip()
         return ToolforgeAction(**json.loads(raw))
 
-    except Exception as exc:
-        print(f"[DEBUG] Model request failed: {exc}", flush=True)
+    except Exception:
         return build_fallback_action(available_tools, current_task)
 
 
@@ -203,67 +217,69 @@ async def main() -> None:
     client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)
 
     env = await ToolforgeEnv.from_docker_image(IMAGE_NAME)
-
-    history: List[str] = []
-    rewards: List[float] = []
-    steps_taken = 0
-    score = 0.0
-    success = False
-
-    log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
+    task_list = get_task_list()
 
     try:
-        result = await env.reset(task_id=TASK_NAME) # OpenENV.reset()
-        obs = result.observation
-        task = obs.current_task
-        available_tools = obs.available_tools
-        # history = obs.history
-        history = [
-            f"EpisodeStart|task_id {getattr(task, 'id', 'unknown')}|difficulty {getattr(task, 'difficulty', 'unknown')}|prompt {getattr(task, 'prompt', '')}"
-        ]
-        last_reward = 0.0
-
-        for step in range(1, MAX_STEPS + 1):
-            if result.done:
-                break
-
-            action = get_model_action(client, step, task.prompt, available_tools, last_reward, history)
-            result = await env.step(action)
-            obs = result.observation
-            task = obs.current_task
-            available_tools = obs.available_tools
-
-            reward = result.reward or 0.0
-            done = result.done
-            error = obs.metadata.get("summary") if isinstance(obs.metadata, dict) else None
-
-            rewards.append(reward)
-            steps_taken = step
-            last_reward = reward
-
-            log_step(
-                step=step,
-                action=action.model_dump_json(),
-                reward=reward,
-                done=done,
-                error=error,
-            )
-
-            history.append(f"{action.model_dump_json()}|Step {step}|reward {reward:+.2f}|task_id {getattr(task, 'id', 'unknown')}|difficulty {getattr(task, 'difficulty', 'unknown')}")
-
-            if done:
-                break
-
-        score = sum(rewards) / MAX_TOTAL_REWARD if MAX_TOTAL_REWARD > 0 else 0.0
-        score = min(max(score, 0.0), 1.0)  # clamp to [0, 1]
-        success = score >= SUCCESS_SCORE_THRESHOLD
-
+        for task_name in task_list:
+            history: List[str] = []
+            rewards: List[float] = []
+            steps_taken = 0
+            score = 0.0
+            success = False
+
+            log_start(task=task_name, env=BENCHMARK, model=MODEL_NAME)
+
+            try:
+                result = await env.reset(task_id=task_name) # OpenENV.reset()
+                obs = result.observation
+                task = obs.current_task
+                available_tools = obs.available_tools
+                history = [
+                    f"EpisodeStart|task_id {getattr(task, 'id', 'unknown')}|difficulty {getattr(task, 'difficulty', 'unknown')}|prompt {getattr(task, 'prompt', '')}"
+                ]
+                last_reward = 0.0
+
+                for step in range(1, MAX_STEPS + 1):
+                    if result.done:
+                        break
+
+                    action = get_model_action(client, step, task.prompt, available_tools, last_reward, history)
+                    result = await env.step(action)
+                    obs = result.observation
+                    task = obs.current_task
+                    available_tools = obs.available_tools
+
+                    reward = result.reward or 0.0
+                    done = result.done
+                    error = obs.metadata.get("summary") if isinstance(obs.metadata, dict) else None
+
+                    rewards.append(reward)
+                    steps_taken = step
+                    last_reward = reward
+
+                    log_step(
+                        step=step,
+                        action=action.model_dump_json(),
+                        reward=reward,
+                        done=done,
+                        error=error,
+                    )
+
+                    history.append(f"{action.model_dump_json()}|Step {step}|reward {reward:+.2f}|task_id {getattr(task, 'id', 'unknown')}|difficulty {getattr(task, 'difficulty', 'unknown')}")
+
+                    if done:
+                        break
+
+                score = sum(rewards) / MAX_TOTAL_REWARD if MAX_TOTAL_REWARD > 0 else 0.0
+                score = max(0.01, min(0.99, score))  # clamp to (0, 1)
+                success = score >= SUCCESS_SCORE_THRESHOLD
+            finally:
+                log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
     finally:
         try:
             await env.close()
-        except Exception as e:
-            print(f"[DEBUG] env.close() error (container cleanup): {e}", flush=True)
-        log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
+        except Exception:
+            pass
 
 
 if __name__ == "__main__":

openenv.yaml

@@ -1,17 +1,18 @@
-spec_version: 1
-name: toolforge_env
-type: space
-runtime: fastapi
-app: server.app:app
-port: 8000
-tasks:
-  - id: easy-deployment-sprints
-    description: "Repetitive service rollout and validation cycles"
-    grader: toolforge_env.graders:EasyGrader
-  - id: medium-traffic-readiness
-    description: "Preparing services for specific traffic goals"
-    grader: toolforge_env.graders:MediumGrader
-  - id: hard-project-legacy-migration
-    description: "Project: Decommissioning Legacy Data Center to Cloud"
-    grader: toolforge_env.graders:HardGrader
-
+spec_version: 1
+name: toolforge_env
+type: space
+runtime: fastapi
+app: server.app:app
+port: 8000
+tasks:
+  - id: easy
+    description: "Repetitive deployment, scaling, and rollback drills with highly repeating tool-call patterns."
+    grader: server.graders.grade_easy
+
+  - id: medium
+    description: "Goal-driven incident response and traffic-readiness scenarios requiring multi-step reasoning."
+    grader: server.graders.grade_medium
+
+  - id: hard
+    description: "Multi-phase infrastructure migration and zero-trust security projects with complex dependencies."
+    grader: server.graders.grade_hard

server/graders.py

@@ -0,0 +1,32 @@
+"""
+Grader functions for the ToolForge environment.
+
+Each grader receives the episode trajectory and returns a score in [0, 1].
+These are referenced by openenv.yaml as 'server.graders.grade_easy' etc.
+"""
+
+import random
+
+
+def _base_grade(*args, **kwargs) -> float:
+    """Placeholder grader that returns a random score.
+    
+    Will be replaced with real grading logic that inspects
+    the episode trajectory once the external grader spec is finalised.
+    """
+    return random.uniform(0.01, 0.99)
+
+
+def grade_easy(*args, **kwargs) -> float:
+    """Grade an easy-tier episode."""
+    return _base_grade(*args, **kwargs)
+
+
+def grade_medium(*args, **kwargs) -> float:
+    """Grade a medium-tier episode."""
+    return _base_grade(*args, **kwargs)
+
+
+def grade_hard(*args, **kwargs) -> float:
+    """Grade a hard-tier episode."""
+    return _base_grade(*args, **kwargs)

server/inputs/simulated/task_selector.py

@@ -1,13 +1,29 @@
-from typing import Dict, Any, List
-
-from server.inputs.simulated.tasks import TASKS
-
-class TaskSelector:
-    def __init__(self):
-        pass
-
-    def next_task_list(self, task_id: str) -> Dict:
-        match = next((item for item in TASKS if item.get("task_id") == task_id), None)
-        if match is None:
-            raise ValueError(f"Unknown task_id: {task_id}")
+from typing import Dict, Any
+
+from server.inputs.simulated.tasks import TASKS
+
+
+TASK_ID_MAP = {
+    "easy": "easy-deployment-sprints",
+    "medium": "medium-traffic-readiness",
+    "hard": "hard-project-legacy-migration",
+}
+
+
+class TaskSelector:
+    def __init__(self):
+        pass
+
+    def next_task_list(self, task_id: str) -> Dict:
+        # Prefer direct internal IDs; if not provided, resolve deterministic aliases.
+        resolved_task_id = task_id
+        direct_match = next((item for item in TASKS if item.get("task_id") == resolved_task_id), None)
+        if direct_match is not None:
+            return direct_match
+
+        resolved_task_id = TASK_ID_MAP.get(task_id, task_id)
+
+        match = next((item for item in TASKS if item.get("task_id") == resolved_task_id), None)
+        if match is None:
+            raise ValueError(f"Unknown task_id: {task_id} (resolved: {resolved_task_id})")
         return match

server/toolforge_env_environment.py

@@ -182,10 +182,10 @@ class ToolforgeEnvironment(Environment):
         self._state = self._create_default_state()
         self._state.episode_id = ep_id
         self._reset_count += 1
-        self.task_id = kwargs.get("task_id", "easy-deployment-sprints")
+        resolved_task_id = task_id or kwargs.get("task_id", "easy")
 
         # subsequent resets ignore incoming values
-        task_list = self._task_selector.next_task_list(self.task_id)
+        task_list = self._task_selector.next_task_list(resolved_task_id)
 
         self._input_provider = self._input_provider_factory(task_list)
         first_task = self._get_next_task_from_generator()