CRITICAL FIX: Wrappers now return full tuples (reward, error, bugs) instead of just floats - fixes env.py unpacking

server/env.py

@@ -207,13 +207,13 @@ def metadata():
         "version": "1.0.0",
         "description": "Config file debugging environment",
         "tasks": [
-            {"id": "task1_json", "name": "JSON Config Debug", "difficulty": "easy", "num_bugs": 2, "has_grader": True, "grader": "server.graders.json_grader:grade_task1"},
-            {"id": "task2_yaml", "name": "YAML Config Debug", "difficulty": "easy", "num_bugs": 2, "has_grader": True, "grader": "server.graders.yaml_grader:grade_task2"},
-            {"id": "task3_dockerfile", "name": "Dockerfile Debug", "difficulty": "medium", "num_bugs": 3, "has_grader": True, "grader": "server.graders.dockerfile_grader:grade_task3"},
-            {"id": "task4_compose", "name": "Docker Compose Debug", "difficulty": "medium", "num_bugs": 4, "has_grader": True, "grader": "server.graders.compose_grader:grade_task4"},
-            {"id": "task5_k8s", "name": "Kubernetes Config Debug", "difficulty": "hard", "num_bugs": 5, "has_grader": True, "grader": "server.graders.k8s_grader:grade_task5"},
-            {"id": "task6_github_actions", "name": "GitHub Actions Debug", "difficulty": "hard", "num_bugs": 5, "has_grader": True, "grader": "server.graders.github_actions_grader:grade_task6"},
-            {"id": "task7_nginx", "name": "Nginx Config Debug", "difficulty": "very_hard", "num_bugs": 6, "has_grader": True, "grader": "server.graders.nginx_grader:grade_task7"},
+            {"id": "task1_json", "name": "JSON Config Debug", "difficulty": "easy", "num_bugs": 2, "has_grader": True, "grader": "server.graders.grader_api:grade_task1"},
+            {"id": "task2_yaml", "name": "YAML Config Debug", "difficulty": "easy", "num_bugs": 2, "has_grader": True, "grader": "server.graders.grader_api:grade_task2"},
+            {"id": "task3_dockerfile", "name": "Dockerfile Debug", "difficulty": "medium", "num_bugs": 3, "has_grader": True, "grader": "server.graders.grader_api:grade_task3"},
+            {"id": "task4_compose", "name": "Docker Compose Debug", "difficulty": "medium", "num_bugs": 4, "has_grader": True, "grader": "server.graders.grader_api:grade_task4"},
+            {"id": "task5_k8s", "name": "Kubernetes Config Debug", "difficulty": "hard", "num_bugs": 5, "has_grader": True, "grader": "server.graders.grader_api:grade_task5"},
+            {"id": "task6_github_actions", "name": "GitHub Actions Debug", "difficulty": "hard", "num_bugs": 5, "has_grader": True, "grader": "server.graders.grader_api:grade_task6"},
+            {"id": "task7_nginx", "name": "Nginx Config Debug", "difficulty": "very_hard", "num_bugs": 6, "has_grader": True, "grader": "server.graders.grader_api:grade_task7"},
         ],
         "action_model": "ConfigDebugAction",
         "observation_model": "ConfigDebugObservation",

server/graders/grader_api.py

@@ -55,46 +55,91 @@ def grade_task1(x):
     """Task 1 (JSON) grader wrapper with strict bounds enforcement."""
     print("[VALIDATOR] grade_task1 called")
     raw = _g1(x)
-    return normalize_reward(raw)
+    # Raw grader returns (reward, error_msg, bugs_fixed)
+    # Normalize the reward but preserve the full tuple structure
+    if isinstance(raw, tuple) and len(raw) >= 3:
+        reward, error_msg, bugs_fixed = raw[0], raw[1], raw[2]
+        normalized_reward = normalize_reward(reward)
+        return normalized_reward, error_msg, bugs_fixed
+    else:
+        # Fallback: if structure unexpected, normalize and return minimal tuple
+        normalized_reward = normalize_reward(raw)
+        return normalized_reward, "Error: unexpected grader output structure", []
 
 
 def grade_task2(x):
     """Task 2 (YAML) grader wrapper with strict bounds enforcement."""
     print("[VALIDATOR] grade_task2 called")
     raw = _g2(x)
-    return normalize_reward(raw)
+    if isinstance(raw, tuple) and len(raw) >= 3:
+        reward, error_msg, bugs_fixed = raw[0], raw[1], raw[2]
+        normalized_reward = normalize_reward(reward)
+        return normalized_reward, error_msg, bugs_fixed
+    else:
+        normalized_reward = normalize_reward(raw)
+        return normalized_reward, "Error: unexpected grader output structure", []
 
 
 def grade_task3(x):
     """Task 3 (Dockerfile) grader wrapper with strict bounds enforcement."""
     print("[VALIDATOR] grade_task3 called")
     raw = _g3(x)
-    return normalize_reward(raw)
+    if isinstance(raw, tuple) and len(raw) >= 3:
+        reward, error_msg, bugs_fixed = raw[0], raw[1], raw[2]
+        normalized_reward = normalize_reward(reward)
+        return normalized_reward, error_msg, bugs_fixed
+    else:
+        normalized_reward = normalize_reward(raw)
+        return normalized_reward, "Error: unexpected grader output structure", []
 
 
 def grade_task4(x):
     """Task 4 (Docker Compose) grader wrapper with strict bounds enforcement."""
     print("[VALIDATOR] grade_task4 called")
     raw = _g4(x)
-    return normalize_reward(raw)
+    if isinstance(raw, tuple) and len(raw) >= 3:
+        reward, error_msg, bugs_fixed = raw[0], raw[1], raw[2]
+        normalized_reward = normalize_reward(reward)
+        return normalized_reward, error_msg, bugs_fixed
+    else:
+        normalized_reward = normalize_reward(raw)
+        return normalized_reward, "Error: unexpected grader output structure", []
 
 
 def grade_task5(x):
     """Task 5 (Kubernetes) grader wrapper with strict bounds enforcement."""
     print("[VALIDATOR] grade_task5 called")
     raw = _g5(x)
-    return normalize_reward(raw)
+    if isinstance(raw, tuple) and len(raw) >= 3:
+        reward, error_msg, bugs_fixed = raw[0], raw[1], raw[2]
+        normalized_reward = normalize_reward(reward)
+        return normalized_reward, error_msg, bugs_fixed
+    else:
+        normalized_reward = normalize_reward(raw)
+        return normalized_reward, "Error: unexpected grader output structure", []
 
 
 def grade_task6(x):
     """Task 6 (GitHub Actions) grader wrapper with strict bounds enforcement."""
     print("[VALIDATOR] grade_task6 called")
     raw = _g6(x)
-    return normalize_reward(raw)
+    if isinstance(raw, tuple) and len(raw) >= 3:
+        reward, error_msg, bugs_fixed = raw[0], raw[1], raw[2]
+        normalized_reward = normalize_reward(reward)
+        return normalized_reward, error_msg, bugs_fixed
+    else:
+        normalized_reward = normalize_reward(raw)
+        return normalized_reward, "Error: unexpected grader output structure", []
 
 
 def grade_task7(x):
     """Task 7 (Nginx) grader wrapper with strict bounds enforcement."""
     print("[VALIDATOR] grade_task7 called")
     raw = _g7(x)
-    return normalize_reward(raw)
+    if isinstance(raw, tuple) and len(raw) >= 3:
+        reward, error_msg, bugs_fixed = raw[0], raw[1], raw[2]
+        normalized_reward = normalize_reward(reward)
+        return normalized_reward, error_msg, bugs_fixed
+    else:
+        normalized_reward = normalize_reward(raw)
+        return normalized_reward, "Error: unexpected grader output structure", []

test_grader_audit.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+Runtime audit of all graders - test actual outputs without validator.
+Tests what the validator will actually receive.
+"""
+
+import sys
+from server.graders.grader_api import (
+    grade_task1, grade_task2, grade_task3, grade_task4,
+    grade_task5, grade_task6, grade_task7,
+)
+from server.tasks import (
+    task1_json, task2_yaml, task3_dockerfile,
+    task4_compose, task5_k8s, task6_github_actions, task7_nginx
+)
+
+# Test data
+TESTS = [
+    (1, grade_task1, task1_json.BROKEN_CONFIG, "JSON"),
+    (2, grade_task2, task2_yaml.BROKEN_CONFIG, "YAML"),
+    (3, grade_task3, task3_dockerfile.BROKEN_CONFIG, "Dockerfile"),
+    (4, grade_task4, task4_compose.BROKEN_CONFIG, "Compose"),
+    (5, grade_task5, task5_k8s.BROKEN_CONFIG, "K8s"),
+    (6, grade_task6, task6_github_actions.BROKEN_CONFIG, "GitHub Actions"),
+    (7, grade_task7, task7_nginx.BROKEN_CONFIG, "Nginx"),
+]
+
+print("=" * 80)
+print("GRADER RUNTIME AUDIT - ALL GRADERS WITH BROKEN CONFIGS")
+print("=" * 80)
+print()
+
+failures = []
+all_valid = True
+
+for task_num, grader_func, broken_config, name in TESTS:
+    task_id = f"task{task_num}_{name.lower().replace(' ', '_')}"
+    
+    print(f"Testing Task {task_num} ({name})...")
+    print(f"  Grader: {grader_func.__name__}")
+    
+    try:
+        result = grader_func(broken_config)
+        
+        # Analyze output structure
+        result_type = type(result).__name__
+        print(f"  Output type: {result_type}")
+        
+        if isinstance(result, tuple) and len(result) >= 3:
+            reward, error_msg, bugs_fixed = result[0], result[1], result[2]
+            print(f"  Reward: {reward}")
+            print(f"  Error msg: {error_msg}")
+            print(f"  Bugs fixed: {bugs_fixed}")
+            
+            # Validate reward
+            if isinstance(reward, float):
+                is_valid = 0 < reward < 1
+                
+                print(f"  Reward is float: True")
+                print(f"  In bounds (0, 1): {is_valid}")
+                print(f"  Exactly 0.0: {reward == 0.0}")
+                print(f"  Exactly 1.0: {reward == 1.0}")
+                print(f"  NaN check: {reward != reward}")
+                print(f"  Inf check: {abs(reward) > 1e308}")
+                
+                if not is_valid:
+                    all_valid = False
+                    failures.append(f"Task {task_num}: Reward {reward} NOT in (0, 1)")
+                    print(f"  ❌ INVALID: Reward {reward} is not in (0, 1) range")
+                else:
+                    print(f"  ✅ Valid")
+            else:
+                all_valid = False
+                failures.append(f"Task {task_num}: Reward is {type(reward).__name__}, not float")
+                print(f"  ❌ INVALID: Reward is {type(reward).__name__}, expected float")
+        else:
+            all_valid = False
+            failures.append(f"Task {task_num}: Expected tuple(reward, error, bugs), got {result_type}")
+            print(f"  ❌ INVALID: Expected tuple(reward, error_msg, bugs_fixed), got {result_type}")
+            print(f"  Value: {result}")
+    
+    except Exception as e:
+        all_valid = False
+        failures.append(f"Task {task_num}: Exception - {type(e).__name__}: {str(e)}")
+        print(f"  ❌ EXCEPTION: {type(e).__name__}: {str(e)}")
+    
+    print()
+
+print("=" * 80)
+print("SUMMARY")
+print("=" * 80)
+print(f"All graders valid: {all_valid}")
+print(f"Total tests: {len(TESTS)}")
+print(f"Passed: {len(TESTS) - len(failures)}")
+print(f"Failed: {len(failures)}")
+
+if failures:
+    print("\nFailures:")
+    for failure in failures:
+        print(f"  - {failure}")
+    sys.exit(1)
+else:
+    print("\n✅ All graders produce valid outputs!")
+    sys.exit(0)