Implement full robustness hardening for grader pipeline

openenv.yaml

@@ -13,66 +13,17 @@ description: >
 
 tasks:
   - id: task1
-    task_id: task1
-    task: task1
-    task_name: task1
     name: Baseline triage
     enabled: true
     grader: graders/task1_grader.py
-    grader_cmd: python graders/task1_grader.py
-    grader_command: python graders/task1_grader.py
-    grader_path: graders/task1_grader.py
-    grader_file: graders/task1_grader.py
-    grader_legacy_path: graders/task1.py
-    graders:
-      - type: python
-        name: default
-        path: graders/task1_grader.py
-        command: python graders/task1_grader.py
   - id: task2
-    task_id: task2
-    task: task2
-    task_name: task2
     name: Moderate pressure triage
     enabled: true
     grader: graders/task2_grader.py
-    grader_cmd: python graders/task2_grader.py
-    grader_command: python graders/task2_grader.py
-    grader_path: graders/task2_grader.py
-    grader_file: graders/task2_grader.py
-    grader_legacy_path: graders/task2.py
-    graders:
-      - type: python
-        name: default
-        path: graders/task2_grader.py
-        command: python graders/task2_grader.py
   - id: task3
-    task_id: task3
-    task: task3
-    task_name: task3
     name: High pressure triage
     enabled: true
     grader: graders/task3_grader.py
-    grader_cmd: python graders/task3_grader.py
-    grader_command: python graders/task3_grader.py
-    grader_path: graders/task3_grader.py
-    grader_file: graders/task3_grader.py
-    grader_legacy_path: graders/task3.py
-    graders:
-      - type: python
-        name: default
-        path: graders/task3_grader.py
-        command: python graders/task3_grader.py
-
-graders:
-  task1: graders/task1_grader.py
-  task2: graders/task2_grader.py
-  task3: graders/task3_grader.py
-
-task_graders:
-  task1: python graders/task1_grader.py
-  task2: python graders/task2_grader.py
-  task3: python graders/task3_grader.py
 
 tags:
   - openenv

triage_env/evaluation/run_llm_agent.py

@@ -1,7 +1,20 @@
+import logging
+import sys
+
 from triage_env.server.triage_env_environment import TriageEnvironment
 from triage_env.agents.llm_agent import LLMAgent
 
 
+if not logging.getLogger().handlers:
+    logging.basicConfig(
+        level=logging.INFO,
+        stream=sys.stderr,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+
+LOGGER = logging.getLogger(__name__)
+
+
 def mock_llm(system_prompt: str, user_prompt: str) -> str:
     # temporary placeholder until real API integration
     _ = system_prompt, user_prompt
@@ -13,17 +26,15 @@ def main():
     agent = LLMAgent(llm_callable=mock_llm)
 
     obs = env.reset()
-    print("Initial Observation:")
-    print(obs.model_dump())
+    LOGGER.info("Initial Observation: %s", obs.model_dump())
 
     while not obs.done:
         action = agent.act(obs)
-        print("\nAction:", action.model_dump())
+        LOGGER.info("Action: %s", action.model_dump())
         obs = env.step(action)
-        print("Observation:", obs.model_dump())
+        LOGGER.info("Observation: %s", obs.model_dump())
 
-    print("\nFinal State:")
-    print(env.state.model_dump())
+    LOGGER.info("Final State: %s", env.state.model_dump())
 
 
 if __name__ == "__main__":

triage_env/graders/common.py

@@ -2,12 +2,16 @@ from __future__ import annotations
 
 import json
 import math
+import logging
 import os
+import multiprocessing as mp
 import sys
 from dataclasses import replace
 from pathlib import Path
 from typing import Any
 
+from jsonschema import ValidationError, validate
+
 # Ensure triage_env package can be imported when graders are executed via file path.
 REPO_ROOT = Path(__file__).resolve().parents[2]
 if str(REPO_ROOT) not in sys.path:
@@ -15,6 +19,7 @@ if str(REPO_ROOT) not in sys.path:
 
 from triage_env.agents.random_agent import RandomAgent
 from triage_env.agents.rl_agents import RLAgent
+from triage_env.agents.base_agent import BaseAgent
 from triage_env.agents.rule_based_agent import RuleBasedAgent
 from triage_env.agents.trained_q_agent import TrainedQAgent
 from triage_env.evaluation.evaluator import evaluate_agent
@@ -23,6 +28,68 @@ from triage_env.tasks import TASK_CONFIGS, TASK_TARGETS
 
 GRADER_VERSION = "v2.2"  # Updated version
 SCORE_EPSILON = 0.001
+GRADE_TIMEOUT_SECONDS = float(os.getenv("TRIAGE_GRADE_TIMEOUT_SECONDS", "300"))
+
+if not logging.getLogger().handlers:
+    logging.basicConfig(
+        level=os.getenv("TRIAGE_LOG_LEVEL", "INFO").upper(),
+        stream=sys.stderr,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+
+LOGGER = logging.getLogger(__name__)
+
+_RESULT_SCHEMA = {
+    "type": "object",
+    "required": ["grader_version", "task", "task_id", "episodes", "score", "reward", "score_range", "components", "signals", "summary"],
+    "properties": {
+        "grader_version": {"type": "string"},
+        "task": {"type": "string"},
+        "task_id": {"type": "string"},
+        "episodes": {"type": "integer", "minimum": 1},
+        "score": {"type": "number", "minimum": SCORE_EPSILON, "maximum": 1.0 - SCORE_EPSILON},
+        "reward": {"type": "number"},
+        "score_range": {
+            "type": "array",
+            "items": {"type": "number"},
+            "minItems": 2,
+            "maxItems": 2,
+        },
+        "components": {"type": "object"},
+        "signals": {"type": "object"},
+        "summary": {"type": "object"},
+    },
+}
+
+
+class SafeAgent(BaseAgent):
+    def __init__(self, wrapped_agent: BaseAgent):
+        self._wrapped_agent = wrapped_agent
+
+    @property
+    def name(self) -> str:
+        return getattr(self._wrapped_agent, "name", self._wrapped_agent.__class__.__name__)
+
+    def reset(self):
+        try:
+            reset = getattr(self._wrapped_agent, "reset", None)
+            if callable(reset):
+                reset()
+        except Exception:
+            LOGGER.exception("SafeAgent reset failed; continuing")
+
+    def act(self, observation):
+        try:
+            return self._wrapped_agent.act(observation)
+        except Exception:
+            LOGGER.exception("Wrapped agent crashed; returning safe wait action")
+            return _safe_wait_action()
+
+
+def _safe_wait_action():
+    from triage_env.models import TriageAction
+
+    return TriageAction(action_type="wait", patient_id=-1)
 
 
 def _resolve_existing_path(candidates: list[Path]) -> Path | None:
@@ -32,6 +99,106 @@ def _resolve_existing_path(candidates: list[Path]) -> Path | None:
     return None
 
 
+def _fallback_grade(task_name: str, episodes: int, reason: str) -> dict[str, Any]:
+    safe_score = _clip_open_01(0.5)
+    return {
+        "grader_version": GRADER_VERSION,
+        "status": "error",
+        "task": task_name,
+        "task_id": task_name,
+        "episodes": episodes,
+        "score": safe_score,
+        "reward": safe_score,
+        "score_range": [0.0, 1.0],
+        "components": {
+            "rollout_achievement": safe_score,
+            "safety_errors": safe_score,
+            "efficiency": safe_score,
+            "task_specific": safe_score,
+        },
+        "signals": {
+            "fallback": 1.0,
+            "error": reason,
+        },
+        "summary": {
+            "task": task_name,
+            "fallback_reason": reason,
+            "success_rate": safe_score,
+            "survival_rate": safe_score,
+            "critical_survival_rate": safe_score,
+            "avg_total_reward": safe_score,
+        },
+    }
+
+
+def _validate_result_schema(result: dict[str, Any]) -> dict[str, Any]:
+    try:
+        validate(instance=result, schema=_RESULT_SCHEMA)
+        return result
+    except ValidationError as exc:
+        LOGGER.error("Grader result failed schema validation: %s", exc)
+        return _fallback_grade(result.get("task", "unknown"), int(result.get("episodes", 1)), f"schema:{exc}")
+
+
+def _grade_task_impl(task_name: str, episodes: int) -> dict[str, Any]:
+    if task_name not in TASK_CONFIGS:
+        raise ValueError(f"Unsupported task: {task_name}")
+
+    task_config = TASK_CONFIGS[task_name]
+    agent, agent_meta = _build_evaluated_agent(task_name)
+    agent = SafeAgent(agent)
+    summary, _ = evaluate_agent(
+        env_class=TriageEnvironment,
+        agent=agent,
+        task=task_name,
+        num_episodes=episodes,
+        max_steps=task_config.max_steps,
+    )
+
+    components = _compute_components(task_name, summary)
+    final_score = _compute_final_score(components)
+
+    return {
+        "grader_version": GRADER_VERSION,
+        "task": task_name,
+        "task_id": task_name,
+        "episodes": episodes,
+        "score": final_score,
+        "reward": final_score,
+        "score_range": [0.0, 1.0],
+        "components": {
+            "rollout_achievement": components["rollout_achievement"],
+            "safety_errors": components["safety_errors"],
+            "efficiency": components["efficiency"],
+            "task_specific": components["task_specific"],
+        },
+        "signals": {
+            "selected_agent": agent_meta.get("selected_agent"),
+            "selected_checkpoint": agent_meta.get("checkpoint"),
+            "selection_reason": agent_meta.get("selection_reason"),
+            "survival_rate": components["survival_rate"],
+            "critical_survival_rate": components["critical_survival_rate"],
+            "success_rate": components["success_rate"],
+            "reward_norm": components["reward_norm"],
+            "invalid_rate": components["invalid_rate"],
+            "stabilization_threshold": components["stabilization_threshold"],
+        },
+        "summary": summary,
+    }
+
+
+def _grade_task_worker(task_name: str, episodes: int, result_queue):
+    try:
+        result_queue.put(_validate_result_schema(_grade_task_impl(task_name, episodes)))
+    except Exception as exc:  # pragma: no cover
+        err = {
+            "type": type(exc).__name__,
+            "message": str(exc),
+            "traceback": "",
+        }
+        result_queue.put(_fallback_grade(task_name, episodes, json.dumps(err, ensure_ascii=True)))
+
+
 def _build_evaluated_agent(task_name: str):
     package_root = Path(__file__).resolve().parents[1]
 
@@ -270,86 +437,38 @@ def _compute_final_score(components: dict[str, float]) -> float:
 
 
 def grade_task(task_name: str, episodes: int = 1) -> dict[str, Any]:
-    try:
-        if task_name not in TASK_CONFIGS:
-            raise ValueError(f"Unsupported task: {task_name}")
-
-        task_config = TASK_CONFIGS[task_name]
-        agent, agent_meta = _build_evaluated_agent(task_name)
-        summary, _ = evaluate_agent(
-            env_class=TriageEnvironment,
-            agent=agent,
-            task=task_name,
-            num_episodes=episodes,
-            max_steps=task_config.max_steps,
-        )
+    ctx = mp.get_context("spawn")
+    result_queue = ctx.Queue(maxsize=1)
+    process = ctx.Process(target=_grade_task_worker, args=(task_name, episodes, result_queue), daemon=True)
 
-        components = _compute_components(task_name, summary)
-        final_score = _compute_final_score(components)
-
-        return {
-            "grader_version": GRADER_VERSION,
-            "task": task_name,
-            "task_id": task_name,
-            "episodes": episodes,
-            "score": final_score,
-            "reward": final_score,
-            "score_range": [0.0, 1.0],
-            "components": {
-                "rollout_achievement": components["rollout_achievement"],
-                "safety_errors": components["safety_errors"],
-                "efficiency": components["efficiency"],
-                "task_specific": components["task_specific"],
-            },
-            "signals": {
-                "selected_agent": agent_meta.get("selected_agent"),
-                "selected_checkpoint": agent_meta.get("checkpoint"),
-                "selection_reason": agent_meta.get("selection_reason"),
-                "survival_rate": components["survival_rate"],
-                "critical_survival_rate": components["critical_survival_rate"],
-                "success_rate": components["success_rate"],
-                "reward_norm": components["reward_norm"],
-                "invalid_rate": components["invalid_rate"],
-                "stabilization_threshold": components["stabilization_threshold"],
-            },
-            "summary": summary,
-        }
-
-    except Exception as e:
-        print(f"❌ grade_task EXCEPTION: {type(e).__name__}: {e}", file=sys.stderr)
-        import traceback
-        traceback.print_exc(file=sys.stderr)
-        
-        safe_score = 0.5
-        
-        # ✅ FIX 3: Restore full component/summary shape (audit: 'weakened schema')
-        return {
-            "grader_version": GRADER_VERSION,
-            "task": task_name,
-            "task_id": task_name,
-            "episodes": episodes,
-            "score": safe_score,
-            "reward": safe_score,
-            "score_range": [0.0, 1.0],
-            "components": {
-                "rollout_achievement": 0.5,
-                "safety_errors": 0.5,
-                "efficiency": 0.5,
-                "task_specific": 0.5,
-            },
-            "signals": {
-                "error_type": type(e).__name__,
-                "error_message": str(e),
-                "status": "fallback_safe_mode",
-                "survival_rate": 0.5,
-                "critical_survival_rate": 0.5,
-                "success_rate": 0.5,
-                "reward_norm": 0.5,
-                "invalid_rate": 0.0,
-                "stabilization_threshold": 0.5,
-            },
-            "summary": {"status": "fallback", "error": str(e)}
-        }
+    try:
+        process.start()
+        process.join(timeout=GRADE_TIMEOUT_SECONDS)
+
+        if process.is_alive():
+            LOGGER.error("Grader timed out after %.1f seconds for task %s", GRADE_TIMEOUT_SECONDS, task_name)
+            process.terminate()
+            process.join(timeout=5)
+            return _fallback_grade(task_name, episodes, f"timeout:{GRADE_TIMEOUT_SECONDS}")
+
+        if result_queue.empty():
+            return _fallback_grade(task_name, episodes, "no-result-from-worker")
+
+        result = result_queue.get_nowait()
+        return _validate_result_schema(result)
+
+    except Exception as exc:  # pragma: no cover
+        LOGGER.exception("Unexpected grader failure for task %s", task_name)
+        return _fallback_grade(task_name, episodes, f"{type(exc).__name__}:{exc}")
+    finally:
+        try:
+            result_queue.close()
+        except Exception:
+            pass
+        try:
+            result_queue.join_thread()
+        except Exception:
+            pass
 
 
 def print_grader_result(result: dict[str, Any]) -> None:

triage_env/openenv.yaml

@@ -1,82 +0,0 @@
-spec_version: 1
-name: criticalops-triage-env
-
-type: space
-runtime: fastapi
-app: server.app:app
-port: 8000
-
-description: >
-  A real-world OpenEnv environment simulating combined medical and military triage scenarios.
-  Agents must prioritize patients, allocate limited resources, and make high-stakes decisions
-  to maximize survival and health outcomes.
-
-tasks:
-  - id: task1
-    task_id: task1
-    task: task1
-    task_name: task1
-    name: Baseline triage
-    enabled: true
-    grader: graders/task1_grader.py
-    grader_cmd: python graders/task1_grader.py
-    grader_command: python graders/task1_grader.py
-    grader_path: graders/task1_grader.py
-    grader_file: graders/task1_grader.py
-    grader_legacy_path: graders/task1.py
-    graders:
-      - type: python
-        name: default
-        path: graders/task1_grader.py
-        command: python graders/task1_grader.py
-  - id: task2
-    task_id: task2
-    task: task2
-    task_name: task2
-    name: Moderate pressure triage
-    enabled: true
-    grader: graders/task2_grader.py
-    grader_cmd: python graders/task2_grader.py
-    grader_command: python graders/task2_grader.py
-    grader_path: graders/task2_grader.py
-    grader_file: graders/task2_grader.py
-    grader_legacy_path: graders/task2.py
-    graders:
-      - type: python
-        name: default
-        path: graders/task2_grader.py
-        command: python graders/task2_grader.py
-  - id: task3
-    task_id: task3
-    task: task3
-    task_name: task3
-    name: High pressure triage
-    enabled: true
-    grader: graders/task3_grader.py
-    grader_cmd: python graders/task3_grader.py
-    grader_command: python graders/task3_grader.py
-    grader_path: graders/task3_grader.py
-    grader_file: graders/task3_grader.py
-    grader_legacy_path: graders/task3.py
-    graders:
-      - type: python
-        name: default
-        path: graders/task3_grader.py
-        command: python graders/task3_grader.py
-
-graders:
-  task1: graders/task1_grader.py
-  task2: graders/task2_grader.py
-  task3: graders/task3_grader.py
-
-task_graders:
-  task1: python graders/task1_grader.py
-  task2: python graders/task2_grader.py
-  task3: python graders/task3_grader.py
-
-tags:
-  - openenv
-  - triage
-  - healthcare
-  - military
-  - decision-making

triage_env/scripts/run_llm_agent.py

@@ -1,6 +1,6 @@
 import argparse
 import logging
-import os
+import sys
 
 from triage_env.agents.llm_agent import LLMAgent
 from triage_env.config import get_llm_config
@@ -9,7 +9,14 @@ from triage_env.evaluation.evaluator import run_single_episode
 from triage_env.server.triage_env_environment import TriageEnvironment
 
 
-logging.basicConfig(level=logging.INFO)
+if not logging.getLogger().handlers:
+    logging.basicConfig(
+        level=logging.INFO,
+        stream=sys.stderr,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+
+LOGGER = logging.getLogger(__name__)
 
 
 def main() -> None:
@@ -20,14 +27,13 @@ def main() -> None:
     args = parser.parse_args()
 
     if not llm_config.api_key:
-        print("API_KEY is not set. LLMAgent may run in fallback mode.")
+        LOGGER.warning("API_KEY is not set. LLMAgent may run in fallback mode.")
     if not llm_config.base_url:
-        print("API_BASE_URL is not set. Requests may not route through the validator proxy.")
+        LOGGER.warning("API_BASE_URL is not set. Requests may not route through the validator proxy.")
 
     env = TriageEnvironment(task=args.task)
     metrics = run_single_episode(env, LLMAgent())
-    print("LLM agent episode metrics:")
-    print(metrics)
+    LOGGER.info("LLM agent episode metrics: %s", metrics)
 
 
 if __name__ == "__main__":

validate-submission.sh

@@ -4,8 +4,7 @@ set -euo pipefail
 # Wrapper entrypoint that forwards to the Python validator.
 # Usage:
 #   ./validate-submission.sh <ping_url> [repo_dir]
-# If repo_dir is omitted and ./triage_env/openenv.yaml exists,
-# it defaults to ./triage_env for convenience.
+# If repo_dir is omitted, validation.py defaults to the repository root.
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 PYTHON_BIN="${PYTHON_BIN:-}"
@@ -23,8 +22,4 @@ if [ -z "$PYTHON_BIN" ]; then
 	fi
 fi
 
-if [ "$#" -eq 1 ] && [ -f "$SCRIPT_DIR/triage_env/openenv.yaml" ]; then
-	exec "$PYTHON_BIN" "$SCRIPT_DIR/validation.py" "$1" "$SCRIPT_DIR/triage_env"
-fi
-
 exec "$PYTHON_BIN" "$SCRIPT_DIR/validation.py" "$@"

validation.py

@@ -165,18 +165,10 @@ def find_docker_context(repo_dir: Path) -> tuple[Path, Path] | None:
 
 
 def find_openenv_dir(repo_dir: Path) -> Path | None:
-    """Find the directory containing openenv.yaml by checking common locations."""
-    # Check root first
+    """Find the directory containing the single source-of-truth openenv.yaml."""
     if (repo_dir / "openenv.yaml").exists():
         return repo_dir
-    
-    # Check common subdirectories
-    for subdir in ["triage_env", "env", "environment", "server"]:
-        candidate = repo_dir / subdir
-        if (candidate / "openenv.yaml").exists():
-            return candidate
-    
-    # If not found, return None
+
     return None
 
 
@@ -225,8 +217,8 @@ def check_step3_openenv_validate(repo_dir: Path) -> None:
     # Find the actual OpenEnv environment directory
     env_dir = find_openenv_dir(repo_dir)
     if env_dir is None:
-        fail_msg("openenv.yaml not found in repo or common subdirectories (triage_env, env, environment, server)")
-        hint(f"Make sure openenv.yaml is in {repo_dir} or a subdirectory like {repo_dir}/triage_env/")
+        fail_msg("openenv.yaml not found at repository root")
+        hint(f"Make sure openenv.yaml is in {repo_dir}")
         stop_at("Step 3")
 
     log(f"  Found openenv.yaml in: {env_dir}")