File size: 3,138 Bytes
cf1808d c256e38 cf1808d c256e38 cf1808d c256e38 cf1808d c256e38 cf1808d c256e38 cf1808d c256e38 cf1808d c256e38 cf1808d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | #!/usr/bin/env python3
"""
Runtime audit of all graders - test actual outputs without validator.
Tests what the validator will actually receive.
"""
import sys
from server.graders.grader_api import (
grade_task1, grade_task2, grade_task3, grade_task4,
grade_task5, grade_task6, grade_task7,
)
from server.tasks import (
task1_json, task2_yaml, task3_dockerfile,
task4_compose, task5_k8s, task6_github_actions, task7_nginx
)
# Test data
TESTS = [
(1, grade_task1, task1_json.BROKEN_CONFIG, "JSON"),
(2, grade_task2, task2_yaml.BROKEN_CONFIG, "YAML"),
(3, grade_task3, task3_dockerfile.BROKEN_CONFIG, "Dockerfile"),
(4, grade_task4, task4_compose.BROKEN_CONFIG, "Compose"),
(5, grade_task5, task5_k8s.BROKEN_CONFIG, "K8s"),
(6, grade_task6, task6_github_actions.BROKEN_CONFIG, "GitHub Actions"),
(7, grade_task7, task7_nginx.BROKEN_CONFIG, "Nginx"),
]
print("=" * 80)
print("GRADER RUNTIME AUDIT - ALL GRADERS WITH BROKEN CONFIGS")
print("Validator Contract: All graders must return FLOAT in (0, 1) ONLY")
print("=" * 80)
print()
failures = []
all_valid = True
for task_num, grader_func, broken_config, name in TESTS:
task_id = f"task{task_num}_{name.lower().replace(' ', '_')}"
print(f"Testing Task {task_num} ({name})...")
print(f" Grader: {grader_func.__name__}")
try:
result = grader_func(broken_config)
# Analyze output - MUST be float only
result_type = type(result).__name__
print(f" Output type: {result_type}")
if isinstance(result, float):
reward = result
is_valid = 0 < reward < 1
print(f" Value: {reward}")
print(f" In bounds (0, 1): {is_valid}")
print(f" Exactly 0.0: {reward == 0.0}")
print(f" Exactly 1.0: {reward == 1.0}")
print(f" NaN check: {reward != reward}")
print(f" Inf check: {abs(reward) > 1e308}")
if not is_valid:
all_valid = False
failures.append(f"Task {task_num}: Reward {reward} NOT in (0, 1)")
print(f" ❌ INVALID: Reward {reward} is not in (0, 1) range")
else:
print(f" ✅ Valid")
else:
all_valid = False
failures.append(f"Task {task_num}: Expected FLOAT, got {result_type}")
print(f" ❌ INVALID: Expected float, got {result_type}")
print(f" Value: {result}")
except Exception as e:
all_valid = False
failures.append(f"Task {task_num}: Exception - {type(e).__name__}: {str(e)}")
print(f" ❌ EXCEPTION: {type(e).__name__}: {str(e)}")
print()
print("=" * 80)
print("SUMMARY")
print("=" * 80)
print(f"All graders valid: {all_valid}")
print(f"Total tests: {len(TESTS)}")
print(f"Passed: {len(TESTS) - len(failures)}")
print(f"Failed: {len(failures)}")
if failures:
print("\nFailures:")
for failure in failures:
print(f" - {failure}")
sys.exit(1)
else:
print("\n✅ All graders return valid FLOATS in (0, 1)!")
sys.exit(0)
|