one more

inference.py

@@ -199,6 +199,13 @@ async def main():
                 log_start(task=current_task, env=benchmark, model=model_name)
 
             result = step_result
+        else:
+            # Loop completed without break — emit [END] for the last active task
+            if task_rewards:
+                task_score = clamp(sum(task_rewards) / len(task_rewards))
+            else:
+                task_score = 0.01
+            log_end(task_score >= 0.5, task_step, task_score, task_rewards)
 
     except Exception as e:
         print(f"[DEBUG] Fatal error: {e}", flush=True)

server/app.py

@@ -1,13 +1,9 @@
 """FastAPI application for ConfigDebugEnv.
 
-Uses OpenEnv's create_fastapi_app() for standard framework compatibility
-(WebSocket sessions, standard endpoints, grader discovery).
+Uses OpenEnv's create_fastapi_app() for standard framework compatibility.
+The framework handles /reset, /step, /state, /health, /schema, /metadata, /ws.
 """
-import json
 import gradio as gr
-from fastapi import Request
-from starlette.middleware.base import BaseHTTPMiddleware
-from starlette.responses import JSONResponse
 
 from openenv.core.env_server import create_fastapi_app
 from server.models import ConfigDebugAction, ConfigDebugObservation, ConfigDebugState
@@ -15,52 +11,13 @@ from server.config_debug_environment import ConfigDebugEnvironment
 from server.tasks.task_registry import get_task, TASK_ORDER
 
 # ---- Create the standard OpenEnv FastAPI app ----
+# The framework registers: /reset, /step, /state, /health, /schema, /metadata, /ws
 app = create_fastapi_app(
     ConfigDebugEnvironment,
     ConfigDebugAction,
     ConfigDebugObservation,
 )
 
-# ---- Remove default routes we need to override ----
-for i, route in enumerate(app.router.routes):
-    if hasattr(route, "path") and route.path == "/reset":
-        app.router.routes.pop(i)
-        print("[APP_INIT] Removed default /reset route for schema fix")
-        break
-
-for i, route in enumerate(app.router.routes):
-    if hasattr(route, "path") and route.path == "/metadata":
-        app.router.routes.pop(i)
-        print("[APP_INIT] Removed default /metadata route for override")
-        break
-
-# ---- Middleware to fix /reset response schema ----
-class ResetSchemaFixMiddleware(BaseHTTPMiddleware):
-    async def dispatch(self, request, call_next):
-        response = await call_next(request)
-        if request.url.path == "/reset" and request.method == "POST":
-            if response.status_code == 200:
-                try:
-                    body = b""
-                    async for chunk in response.body_iterator:
-                        body += chunk
-                    data = json.loads(body)
-                    if isinstance(data, dict) and "observation" in data:
-                        fixed_data = {
-                            "observation": data["observation"],
-                            "done": data.get("done", False),
-                            "reward": 0.0,
-                            "metadata": data.get("metadata", {}),
-                            "info": {}
-                        }
-                        print("[MIDDLEWARE] Fixed /reset response schema - added base fields")
-                        return JSONResponse(fixed_data, status_code=200)
-                except Exception as e:
-                    print(f"[MIDDLEWARE] Error fixing reset response: {e}")
-        return response
-
-app.add_middleware(ResetSchemaFixMiddleware)
-
 # ---- Startup Diagnostics ----
 print("[APP_INIT] ConfigDebugEnvironment initialization started")
 print(f"[APP_INIT] Loaded {len(TASK_ORDER)} tasks: {TASK_ORDER}")
@@ -72,108 +29,13 @@ for task_id in TASK_ORDER:
         print(f"[APP_INIT] ERROR loading task '{task_id}': {str(e)}")
 
 
-# ---- Custom endpoints ----
+# ---- Custom endpoints (non-conflicting with framework) ----
 
 @app.get("/info")
 def info():
     return {"name": "ConfigDebugEnv", "version": "1.0.0", "status": "running"}
 
 
-@app.get("/health")
-def health():
-    return {"status": "healthy"}
-
-
-@app.get("/metadata")
-def metadata():
-    """Metadata endpoint with grader paths for validator discovery."""
-    print("[VALIDATOR] GET /metadata called")
-    
-    task_grader_map = {
-        "task1_json": "Task1Grader",
-        "task2_yaml": "Task2Grader",
-        "task3_dockerfile": "Task3Grader",
-        "task4_compose": "Task4Grader",
-        "task5_k8s": "Task5Grader",
-        "task6_github_actions": "Task6Grader",
-        "task7_nginx": "Task7Grader",
-    }
-    
-    return {
-        "name": "ConfigDebugEnvironment",
-        "description": "An environment for training AI agents to debug broken configuration files",
-        "version": "1.0.0",
-        "tasks": [
-            {
-                "id": tid,
-                "has_grader": True,
-                "grader": f"server.graders.grader_api:{task_grader_map[tid]}",
-            }
-            for tid in TASK_ORDER
-        ],
-    }
-
-
-@app.post("/reset")
-async def reset_env(request: Request):
-    """Override /reset endpoint to return correct OpenEnv contract schema."""
-    print("[VALIDATOR] POST /reset called - CUSTOM OVERRIDE")
-    try:
-        env = ConfigDebugEnvironment()
-        observation = env.reset()
-        fixed_response = {
-            "observation": observation.model_dump(),
-            "done": False,
-            "reward": 0.0,
-            "metadata": {},
-            "info": {}
-        }
-        print("[VALIDATOR] Reset response formatted with base fields")
-        return fixed_response
-    except Exception as e:
-        print(f"[VALIDATOR] Error in custom reset: {e}")
-        raise
-
-
-@app.post("/grader")
-async def grader_endpoint(request: Request):
-    """Score a submitted config for a specific task without a full episode.
-    The validator calls this to verify each task has a working grader
-    with scores strictly between 0 and 1."""
-    print("[VALIDATOR] POST /grader called")
-    try:
-        body = await request.json()
-        task_id = body.get("task_id", TASK_ORDER[0])
-        submitted_config = body.get("submitted_config",
-            body.get("action", {}).get("fixed_config", "{}"))
-
-        from server.graders.grader_api import (
-            Task1Grader, Task2Grader, Task3Grader,
-            Task4Grader, Task5Grader, Task6Grader, Task7Grader,
-        )
-
-        grader_map = {
-            "task1_json": Task1Grader(),
-            "task2_yaml": Task2Grader(),
-            "task3_dockerfile": Task3Grader(),
-            "task4_compose": Task4Grader(),
-            "task5_k8s": Task5Grader(),
-            "task6_github_actions": Task6Grader(),
-            "task7_nginx": Task7Grader(),
-        }
-
-        grader = grader_map.get(task_id)
-        if grader is None:
-            return {"error": f"Unknown task_id: {task_id}", "score": 0.01}
-
-        score = grader.grade(submitted_config)
-        print(f"[GRADER] task={task_id} score={score}")
-        return {"task_id": task_id, "score": score, "has_grader": True}
-    except Exception as e:
-        print(f"[GRADER] Error: {e}")
-        return {"error": str(e), "score": 0.01}
-
-
 @app.get("/tasks")
 def tasks():
     task_grader_map = {
@@ -203,15 +65,6 @@ def tasks():
     }
 
 
-@app.get("/schema")
-def schema():
-    return {
-        "action": ConfigDebugAction.model_json_schema(),
-        "observation": ConfigDebugObservation.model_json_schema(),
-        "state": ConfigDebugState.model_json_schema(),
-    }
-
-
 # ---- Gradio Web UI ----
 
 _ui_env = ConfigDebugEnvironment()

server/config_debug_environment.py

@@ -23,6 +23,7 @@ class ConfigDebugEnvironment(Environment):
     SUPPORTS_CONCURRENT_SESSIONS = True
 
     def __init__(self):
+        super().__init__()
         self._init_episode()
 
     def _init_episode(self):
@@ -33,7 +34,7 @@ class ConfigDebugEnvironment(Environment):
         self._done = False
         self.tasks_completed: list = []
         self.bugs_found_so_far = 0
-        self.previous_reward = 0.0
+        self.previous_reward = None
         self.current_error_message: Optional[str] = None
         self.current_broken_config: Optional[str] = None
         self._episode_id = str(uuid4())
@@ -164,7 +165,7 @@ class ConfigDebugEnvironment(Environment):
             difficulty=task.difficulty,
             num_bugs=task.num_bugs,
             bugs_found_so_far=self.bugs_found_so_far,
-            previous_reward=self.previous_reward,
+            previous_reward=self.previous_reward if self.previous_reward is not None else 0.01,
             done=self._done,
             reward=self.previous_reward,
         )

test_env.py

@@ -1,21 +0,0 @@
-from server.config_debug_environment import ConfigDebugEnvironment
-from server.models import ConfigDebugAction
-from server.tasks.task_registry import get_task
-
-env = ConfigDebugEnvironment()
-
-# ONLY ONE RESET
-obs = env.reset()
-
-for i in range(3):
-    print(f"\n--- Step {i+1} ---")
-
-    task = get_task(obs.task_id)
-    correct_output = task.ground_truth
-
-    action = ConfigDebugAction(fixed_config=correct_output)
-    obs = env.step(action)
-
-    print("Task ID:", obs.task_id)
-    print("Reward:", obs.reward)
-    print("Done:", obs.done)

test_grader_audit.py

@@ -1,95 +0,0 @@
-#!/usr/bin/env python3
-"""
-Runtime audit of all graders - test actual outputs without validator.
-Tests what the validator will actually receive.
-"""
-
-import sys
-from server.graders.grader_api import (
-    grade_task1, grade_task2, grade_task3, grade_task4,
-    grade_task5, grade_task6, grade_task7,
-)
-from server.tasks import (
-    task1_json, task2_yaml, task3_dockerfile,
-    task4_compose, task5_k8s, task6_github_actions, task7_nginx
-)
-
-# Test data
-TESTS = [
-    (1, grade_task1, task1_json.BROKEN_CONFIG, "JSON"),
-    (2, grade_task2, task2_yaml.BROKEN_CONFIG, "YAML"),
-    (3, grade_task3, task3_dockerfile.BROKEN_CONFIG, "Dockerfile"),
-    (4, grade_task4, task4_compose.BROKEN_CONFIG, "Compose"),
-    (5, grade_task5, task5_k8s.BROKEN_CONFIG, "K8s"),
-    (6, grade_task6, task6_github_actions.BROKEN_CONFIG, "GitHub Actions"),
-    (7, grade_task7, task7_nginx.BROKEN_CONFIG, "Nginx"),
-]
-
-print("=" * 80)
-print("GRADER RUNTIME AUDIT - ALL GRADERS WITH BROKEN CONFIGS")
-print("Validator Contract: All graders must return FLOAT in (0, 1) ONLY")
-print("=" * 80)
-print()
-
-failures = []
-all_valid = True
-
-for task_num, grader_func, broken_config, name in TESTS:
-    task_id = f"task{task_num}_{name.lower().replace(' ', '_')}"
-    
-    print(f"Testing Task {task_num} ({name})...")
-    print(f"  Grader: {grader_func.__name__}")
-    
-    try:
-        result = grader_func(broken_config)
-        
-        # Analyze output - MUST be float only
-        result_type = type(result).__name__
-        print(f"  Output type: {result_type}")
-        
-        if isinstance(result, float):
-            reward = result
-            is_valid = 0 < reward < 1
-            
-            print(f"  Value: {reward}")
-            print(f"  In bounds (0, 1): {is_valid}")
-            print(f"  Exactly 0.0: {reward == 0.0}")
-            print(f"  Exactly 1.0: {reward == 1.0}")
-            print(f"  NaN check: {reward != reward}")
-            print(f"  Inf check: {abs(reward) > 1e308}")
-            
-            if not is_valid:
-                all_valid = False
-                failures.append(f"Task {task_num}: Reward {reward} NOT in (0, 1)")
-                print(f"  ❌ INVALID: Reward {reward} is not in (0, 1) range")
-            else:
-                print(f"  ✅ Valid")
-        else:
-            all_valid = False
-            failures.append(f"Task {task_num}: Expected FLOAT, got {result_type}")
-            print(f"  ❌ INVALID: Expected float, got {result_type}")
-            print(f"  Value: {result}")
-    
-    except Exception as e:
-        all_valid = False
-        failures.append(f"Task {task_num}: Exception - {type(e).__name__}: {str(e)}")
-        print(f"  ❌ EXCEPTION: {type(e).__name__}: {str(e)}")
-    
-    print()
-
-print("=" * 80)
-print("SUMMARY")
-print("=" * 80)
-print(f"All graders valid: {all_valid}")
-print(f"Total tests: {len(TESTS)}")
-print(f"Passed: {len(TESTS) - len(failures)}")
-print(f"Failed: {len(failures)}")
-
-if failures:
-    print("\nFailures:")
-    for failure in failures:
-        print(f"  - {failure}")
-    sys.exit(1)
-else:
-    print("\n✅ All graders return valid FLOATS in (0, 1)!")
-    sys.exit(0)

test_k8s_multistep.py

@@ -1,88 +0,0 @@
-from server.graders.k8s_grader import grade_task5
-
-# Test 1: Only fix replicas
-config1 = """apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: my-app
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: my-app
-  template:
-    metadata:
-      labels:
-        app: my-app
-    spec:
-      containers:
-        - name: my-container
-          image: nginx
-          ports:
-            - containerPort: "80"
-          resources:
-            limits:
-              cpu: "500"
-"""
-
-reward1, msg1, fixed1 = grade_task5(config1)
-print(f'Step 1 (replicas fixed): Reward={reward1}, Fixed={fixed1}')
-print(f'  Error: {msg1}\n')
-
-# Test 2: Fix replicas + port
-config2 = """apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: my-app
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: my-app
-  template:
-    metadata:
-      labels:
-        app: my-app
-    spec:
-      containers:
-        - name: my-container
-          image: nginx
-          ports:
-            - containerPort: 80
-          resources:
-            limits:
-              cpu: "500"
-"""
-
-reward2, msg2, fixed2 = grade_task5(config2)
-print(f'Step 2 (replicas + port fixed): Reward={reward2}, Fixed={fixed2}')
-print(f'  Error: {msg2}\n')
-
-# Test 3: Fix everything
-config3 = """apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: my-app
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: my-app
-  template:
-    metadata:
-      labels:
-        app: my-app
-    spec:
-      containers:
-        - name: my-container
-          image: nginx
-          ports:
-            - containerPort: 80
-          resources:
-            limits:
-              cpu: "500m"
-"""
-
-reward3, msg3, fixed3 = grade_task5(config3)
-print(f'Step 3 (all fixed): Reward={reward3}, Fixed={fixed3}')
-print(f'  Message: {msg3}')

test_nginx_multistep.py

@@ -1,72 +0,0 @@
-from server.graders.nginx_grader import grade_task7
-
-# Test 1: Only fix syntax (listen and error_log semicolons)
-config1 = """events {}
-
-http {
-    server {
-        listen 80;
-
-        location / {
-            proxy_pass localhost:3000
-        }
-
-        location /api {
-            proxy_pass http://localhost:5000
-        }
-
-        error_log logs/error.log;
-    }
-}"""
-
-reward1, msg1, fixed1 = grade_task7(config1)
-print(f'Step 1 (syntax fixed): Reward={reward1}, Fixed={fixed1}')
-print(f'  Error: {msg1}\n')
-
-# Test 2: Fix syntax + proxy_pass (add http://)
-config2 = """events {}
-
-http {
-    server {
-        listen 80;
-
-        location / {
-            proxy_pass http://localhost:3000;
-        }
-
-        location /api {
-            proxy_pass http://localhost:5000
-        }
-
-        error_log logs/error.log;
-    }
-}"""
-
-reward2, msg2, fixed2 = grade_task7(config2)
-print(f'Step 2 (syntax + proxy_pass fixed): Reward={reward2}, Fixed={fixed2}')
-print(f'  Error: {msg2}\n')
-
-# Test 3: Fix everything (add /api/ and headers)
-config3 = """events {}
-
-http {
-    server {
-        listen 80;
-
-        location / {
-            proxy_pass http://localhost:3000;
-        }
-
-        location /api/ {
-            proxy_pass http://localhost:5000;
-            proxy_set_header Host $host;
-            proxy_set_header X-Real-IP $remote_addr;
-        }
-
-        error_log logs/error.log;
-    }
-}"""
-
-reward3, msg3, fixed3 = grade_task7(config3)
-print(f'Step 3 (all fixed): Reward={reward3}, Fixed={fixed3}')
-print(f'  Message: {msg3}')

test_task_graders.py

@@ -1,84 +0,0 @@
-#!/usr/bin/env python3
-"""
-Local Test Script: Verify all tasks/graders meet benchmark quality standards.
-
-This script runs the mentor's quality checklist:
-- Broken config should score ~0.05 reward
-- Ground truth should score ~0.95 reward
-- bugs_fixed list should reflect actual fixes
-- No task should return (reward=0.95, bugs=[]) on broken config (indicates broken grader)
-"""
-
-from server.tasks.task_registry import TASK_ORDER, get_task
-
-
-def test_all_tasks():
-    """Test all tasks in TASK_ORDER."""
-    print("=" * 80)
-    print("BENCHMARK QUALITY TEST - ALL TASKS")
-    print("=" * 80)
-    
-    all_passed = True
-    
-    for task_id in TASK_ORDER:
-        print(f"\n[TEST] {task_id}")
-        print("-" * 80)
-        
-        task = get_task(task_id)
-        
-        # Test 1: Broken config should score low
-        broken_reward, broken_msg, broken_bugs = task.grader(task.broken_config)
-        print(f"  BROKEN CONFIG:")
-        print(f"    Reward: {broken_reward:.2f} (expected ~0.05)")
-        print(f"    Bugs Fixed: {len(broken_bugs)} (expected 0-1)")
-        print(f"    Message: {broken_msg[:60]}...")
-        
-        broken_ok = 0.01 <= broken_reward <= 0.99  # Must be strictly (0, 1)
-        if not broken_ok:
-            print(f"    [FAIL] Broken config reward out of valid range (0.01-0.99)!")
-            all_passed = False
-        else:
-            print(f"    [PASS]")
-        
-        # Test 2: Ground truth should score high
-        truth_reward, truth_msg, truth_bugs = task.grader(task.ground_truth)
-        print(f"\n  GROUND TRUTH:")
-        print(f"    Reward: {truth_reward:.2f} (expected 0.85-0.99)")
-        print(f"    Bugs Fixed: {len(truth_bugs)} (expected {task.num_bugs})")
-        print(f"    Message: {truth_msg[:60]}...")
-        
-        truth_ok = 0.85 <= truth_reward <= 0.99  # Should be in valid high range
-        if not truth_ok:
-            print(f"    [FAIL] Ground truth reward out of expected range!")
-            all_passed = False
-        else:
-            print(f"    [PASS]")
-        
-        # Test 3: Check for grader logic issues
-        if broken_reward >= 0.95 and len(broken_bugs) == 0:
-            print(f"\n  [WARNING] Grader may be broken (high reward on broken config)")
-            all_passed = False
-        
-        if truth_reward < 0.85:
-            print(f"\n  [WARNING] Ground truth not scoring highly enough")
-            all_passed = False
-        
-        # Task metadata
-        print(f"\n  Task Metadata:")
-        print(f"    Difficulty: {task.difficulty}")
-        print(f"    Bugs: {task.num_bugs}")
-        print(f"    Type: {task.file_type}")
-    
-    print("\n" + "=" * 80)
-    if all_passed:
-        print("[SUCCESS] ALL TESTS PASSED - Ready for resubmission")
-    else:
-        print("[FAILED] SOME TESTS FAILED - Review above")
-    print("=" * 80)
-    
-    return all_passed
-
-
-if __name__ == "__main__":
-    success = test_all_tasks()
-    exit(0 if success else 1)

test_validator_simulation.py

@@ -1,88 +0,0 @@
-"""
-Simulates EXACTLY what the Scaler validator does:
-1. Import grader classes from openenv.yaml paths
-2. Instantiate each class
-3. Call .grade(env) or .grade(None)
-4. Check score is strictly in (0.01, 0.90)
-
-Also tests genuine grading: broken configs get LOW scores, fixed configs get HIGH scores.
-
-Run: python test_validator_simulation.py
-"""
-import sys
-import os
-
-# Add project root to path
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-def test_grader_classes():
-    print("=" * 60)
-    print("VALIDATOR SIMULATION: Testing grader classes")
-    print("=" * 60)
-    
-    from server.graders.grader_api import (
-        Task1Grader, Task2Grader, Task3Grader,
-        Task4Grader, Task5Grader, Task6Grader, Task7Grader,
-    )
-    from server.config_debug_environment import ConfigDebugEnvironment
-    from server.tasks.task_registry import get_task, TASK_ORDER
-    
-    grader_classes = {
-        "task1_json": Task1Grader,
-        "task2_yaml": Task2Grader,
-        "task3_dockerfile": Task3Grader,
-        "task4_compose": Task4Grader,
-        "task5_k8s": Task5Grader,
-        "task6_github_actions": Task6Grader,
-        "task7_nginx": Task7Grader,
-    }
-    
-    all_pass = True
-    
-    for task_id, GraderClass in grader_classes.items():
-        grader = GraderClass()
-        task = get_task(task_id)
-        
-        # Test 1: .grade(None) — validator may call with None
-        # Should grade the BROKEN config → low/medium score
-        score_none = grader.grade(None)
-        ok_none = 0.0 < score_none < 1.0
-        
-        # Test 2: .grade(env) — validator passes fresh environment
-        env = ConfigDebugEnvironment()
-        env.reset(task_id=task_id)
-        score_env = grader.grade(env)
-        ok_env = 0.0 < score_env < 1.0
-        
-        # Test 3: .grade(ground_truth_string) — direct string (correct answer)
-        score_gt = grader.grade(task.ground_truth)
-        ok_gt = 0.0 < score_gt < 1.0
-        
-        # Test 4: .grade(broken_config) — direct string (broken)
-        score_broken = grader.grade(task.broken_config)
-        ok_broken = 0.0 < score_broken < 1.0
-        
-        # Verify genuine grading: fixed > broken
-        genuine = score_gt > score_broken or score_gt == score_broken  # fixed should score higher
-        
-        status = "PASS" if (ok_none and ok_env and ok_gt and ok_broken) else "FAIL"
-        if not (ok_none and ok_env and ok_gt and ok_broken):
-            all_pass = False
-        
-        print(f"\n[{status}] {task_id}")
-        print(f"  .grade(None)           = {score_none:.4f}  {'OK' if ok_none else 'OUT OF RANGE!'}")
-        print(f"  .grade(env)            = {score_env:.4f}  {'OK' if ok_env else 'OUT OF RANGE!'}")
-        print(f"  .grade(ground_truth)   = {score_gt:.4f}  (correct answer)")
-        print(f"  .grade(broken_config)  = {score_broken:.4f}  (broken input)")
-        print(f"  Genuine grading: fixed({score_gt:.2f}) >= broken({score_broken:.2f}) = {genuine}")
-    
-    print("\n" + "=" * 60)
-    if all_pass:
-        print("ALL TASKS PASSED — all scores strictly in (0, 1)")
-    else:
-        print("SOME TASKS FAILED — fix before pushing!")
-    print("=" * 60)
-
-
-if __name__ == "__main__":
-    test_grader_classes()