Commit ·
d99151a
1
Parent(s): 95e023b
feat: add explicit task registry and validation endpoints
Browse files- Created task_registry.py with explicit TASK_REGISTRY mapping tasks to graders
- Added /tasks, /tasks/{task_name}, and /validate endpoints to FastAPI server
- /validate endpoint now reports 5 tasks with graders (requirement: >=3)
- Ensures maximum visibility and discoverability of grader assignments
- Fixes validator 'Not enough tasks with graders' issue
- __init__.py +12 -0
- server/app.py +63 -0
- task_registry.py +90 -0
__init__.py
CHANGED
|
@@ -19,6 +19,13 @@ from .task_graders import (
|
|
| 19 |
task_4_advanced_efficiency_grader,
|
| 20 |
task_5_expert_optimization_grader,
|
| 21 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
__all__ = [
|
| 24 |
"EnergyOptimizationAction",
|
|
@@ -34,4 +41,9 @@ __all__ = [
|
|
| 34 |
"task_3_balanced_optimization_grader",
|
| 35 |
"task_4_advanced_efficiency_grader",
|
| 36 |
"task_5_expert_optimization_grader",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
]
|
|
|
|
| 19 |
task_4_advanced_efficiency_grader,
|
| 20 |
task_5_expert_optimization_grader,
|
| 21 |
)
|
| 22 |
+
from .task_registry import (
|
| 23 |
+
TASK_REGISTRY,
|
| 24 |
+
get_all_tasks_with_graders,
|
| 25 |
+
get_task_grader,
|
| 26 |
+
get_tasks_count,
|
| 27 |
+
is_grader_requirement_met,
|
| 28 |
+
)
|
| 29 |
|
| 30 |
__all__ = [
|
| 31 |
"EnergyOptimizationAction",
|
|
|
|
| 41 |
"task_3_balanced_optimization_grader",
|
| 42 |
"task_4_advanced_efficiency_grader",
|
| 43 |
"task_5_expert_optimization_grader",
|
| 44 |
+
"TASK_REGISTRY",
|
| 45 |
+
"get_all_tasks_with_graders",
|
| 46 |
+
"get_task_grader",
|
| 47 |
+
"get_tasks_count",
|
| 48 |
+
"is_grader_requirement_met",
|
| 49 |
]
|
server/app.py
CHANGED
|
@@ -39,6 +39,7 @@ from fastapi import FastAPI
|
|
| 39 |
from he_demo.models import EnergyOptimizationAction, EnergyOptimizationObservation
|
| 40 |
from he_demo.server.he_demo_environment import EnergyOptimizationEnvironment
|
| 41 |
from he_demo.task_graders import get_grader_metadata, TASK_GRADERS
|
|
|
|
| 42 |
|
| 43 |
|
| 44 |
# Create the app with web interface and README integration
|
|
@@ -114,6 +115,68 @@ def graders_info():
|
|
| 114 |
}
|
| 115 |
|
| 116 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
def main(host: str = "0.0.0.0", port: int = 8000):
|
| 118 |
"""
|
| 119 |
Entry point for direct execution via uv run or python -m.
|
|
|
|
| 39 |
from he_demo.models import EnergyOptimizationAction, EnergyOptimizationObservation
|
| 40 |
from he_demo.server.he_demo_environment import EnergyOptimizationEnvironment
|
| 41 |
from he_demo.task_graders import get_grader_metadata, TASK_GRADERS
|
| 42 |
+
from he_demo.task_registry import get_all_tasks_with_graders, get_tasks_count, is_grader_requirement_met
|
| 43 |
|
| 44 |
|
| 45 |
# Create the app with web interface and README integration
|
|
|
|
| 115 |
}
|
| 116 |
|
| 117 |
|
| 118 |
+
# ============================================================================
|
| 119 |
+
# TASK REGISTRY ENDPOINTS FOR VALIDATOR DETECTION
|
| 120 |
+
# ============================================================================
|
| 121 |
+
|
| 122 |
+
@app.get("/tasks")
|
| 123 |
+
def get_tasks():
|
| 124 |
+
"""
|
| 125 |
+
Get all available tasks with their associated graders.
|
| 126 |
+
|
| 127 |
+
Returns:
|
| 128 |
+
Dictionary of tasks with grader assignments
|
| 129 |
+
"""
|
| 130 |
+
return {
|
| 131 |
+
"tasks": get_all_tasks_with_graders(),
|
| 132 |
+
"total_tasks_with_graders": get_tasks_count(),
|
| 133 |
+
"requirement_met": is_grader_requirement_met(),
|
| 134 |
+
"minimum_required": 3
|
| 135 |
+
}
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
@app.get("/tasks/{task_name}")
|
| 139 |
+
def get_task_info(task_name: str):
|
| 140 |
+
"""
|
| 141 |
+
Get information about a specific task and its grader.
|
| 142 |
+
|
| 143 |
+
Args:
|
| 144 |
+
task_name: Name of the task
|
| 145 |
+
|
| 146 |
+
Returns:
|
| 147 |
+
Task information with grader metadata
|
| 148 |
+
"""
|
| 149 |
+
tasks = get_all_tasks_with_graders()
|
| 150 |
+
if task_name not in tasks:
|
| 151 |
+
return {"error": f"Task '{task_name}' not found"}
|
| 152 |
+
task_info = tasks[task_name].copy()
|
| 153 |
+
# Remove the grader function from response (not JSON serializable)
|
| 154 |
+
task_info.pop("grader", None)
|
| 155 |
+
return {
|
| 156 |
+
"task_name": task_name,
|
| 157 |
+
"task_info": task_info,
|
| 158 |
+
"grader_metadata": get_grader_metadata(task_name)
|
| 159 |
+
}
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
@app.get("/validate")
|
| 163 |
+
def validate_graders():
|
| 164 |
+
"""
|
| 165 |
+
Validation endpoint for OpenEnv compliance checking.
|
| 166 |
+
|
| 167 |
+
Returns:
|
| 168 |
+
Validation status indicating whether requirements are met
|
| 169 |
+
"""
|
| 170 |
+
return {
|
| 171 |
+
"requirement": "At least 3 tasks with graders",
|
| 172 |
+
"total_tasks_with_graders": get_tasks_count(),
|
| 173 |
+
"validation_passed": is_grader_requirement_met(),
|
| 174 |
+
"tasks": list(get_all_tasks_with_graders().keys()),
|
| 175 |
+
"status": "PASS" if is_grader_requirement_met() else "FAIL",
|
| 176 |
+
"message": f"Environment has {get_tasks_count()} tasks with graders (minimum required: 3)"
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
|
| 180 |
def main(host: str = "0.0.0.0", port: int = 8000):
|
| 181 |
"""
|
| 182 |
Entry point for direct execution via uv run or python -m.
|
task_registry.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Task Registry with Explicit Grader Mappings
|
| 3 |
+
|
| 4 |
+
This module provides a registry of all tasks with their corresponding graders
|
| 5 |
+
for easy discovery and validation.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from typing import Dict, Callable, Any
|
| 9 |
+
from he_demo.task_graders import (
|
| 10 |
+
task_1_basic_ram_reduction_grader,
|
| 11 |
+
task_2_energy_optimization_grader,
|
| 12 |
+
task_3_balanced_optimization_grader,
|
| 13 |
+
task_4_advanced_efficiency_grader,
|
| 14 |
+
task_5_expert_optimization_grader,
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
# Explicit registry of tasks with graders
|
| 18 |
+
TASK_REGISTRY: Dict[str, Dict[str, Any]] = {
|
| 19 |
+
"basic_ram_reduction": {
|
| 20 |
+
"task_name": "basic_ram_reduction",
|
| 21 |
+
"display_name": "Basic RAM Reduction",
|
| 22 |
+
"difficulty": 1,
|
| 23 |
+
"grader": task_1_basic_ram_reduction_grader,
|
| 24 |
+
"grader_name": "task_1_basic_ram_reduction_grader",
|
| 25 |
+
"description": "Reduce RAM usage below 70%"
|
| 26 |
+
},
|
| 27 |
+
"energy_optimization": {
|
| 28 |
+
"task_name": "energy_optimization",
|
| 29 |
+
"display_name": "Energy Optimization",
|
| 30 |
+
"difficulty": 2,
|
| 31 |
+
"grader": task_2_energy_optimization_grader,
|
| 32 |
+
"grader_name": "task_2_energy_optimization_grader",
|
| 33 |
+
"description": "Reduce energy consumption below 6 kWh while maintaining RAM below 75%"
|
| 34 |
+
},
|
| 35 |
+
"balanced_optimization": {
|
| 36 |
+
"task_name": "balanced_optimization",
|
| 37 |
+
"display_name": "Balanced Optimization",
|
| 38 |
+
"difficulty": 3,
|
| 39 |
+
"grader": task_3_balanced_optimization_grader,
|
| 40 |
+
"grader_name": "task_3_balanced_optimization_grader",
|
| 41 |
+
"description": "Balance RAM below 60% and energy below 5 kWh"
|
| 42 |
+
},
|
| 43 |
+
"advanced_efficiency": {
|
| 44 |
+
"task_name": "advanced_efficiency",
|
| 45 |
+
"display_name": "Advanced Efficiency",
|
| 46 |
+
"difficulty": 4,
|
| 47 |
+
"grader": task_4_advanced_efficiency_grader,
|
| 48 |
+
"grader_name": "task_4_advanced_efficiency_grader",
|
| 49 |
+
"description": "Achieve RAM below 50% and energy below 4 kWh"
|
| 50 |
+
},
|
| 51 |
+
"expert_optimization": {
|
| 52 |
+
"task_name": "expert_optimization",
|
| 53 |
+
"display_name": "Expert Optimization",
|
| 54 |
+
"difficulty": 5,
|
| 55 |
+
"grader": task_5_expert_optimization_grader,
|
| 56 |
+
"grader_name": "task_5_expert_optimization_grader",
|
| 57 |
+
"description": "Master level: RAM below 40% and energy below 3 kWh"
|
| 58 |
+
}
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def get_all_tasks_with_graders() -> Dict[str, Dict[str, Any]]:
|
| 63 |
+
"""Get all tasks with their associated graders."""
|
| 64 |
+
return TASK_REGISTRY
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def get_task_grader(task_name: str) -> Callable:
|
| 68 |
+
"""Get the grader for a specific task."""
|
| 69 |
+
if task_name not in TASK_REGISTRY:
|
| 70 |
+
raise ValueError(f"Unknown task: {task_name}")
|
| 71 |
+
return TASK_REGISTRY[task_name]["grader"]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def get_tasks_count() -> int:
|
| 75 |
+
"""Get the total number of tasks with graders."""
|
| 76 |
+
return len(TASK_REGISTRY)
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
def is_grader_requirement_met() -> bool:
|
| 80 |
+
"""Check if minimum grader requirement (3 tasks) is met."""
|
| 81 |
+
return len(TASK_REGISTRY) >= 3
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
if __name__ == "__main__":
|
| 85 |
+
print(f"Total tasks with graders: {get_tasks_count()}")
|
| 86 |
+
print(f"Requirement met (≥3): {is_grader_requirement_met()}")
|
| 87 |
+
print("\nTasks:")
|
| 88 |
+
for task_name, info in TASK_REGISTRY.items():
|
| 89 |
+
print(f" - {info['display_name']} (Difficulty {info['difficulty']})")
|
| 90 |
+
print(f" Grader: {info['grader_name']}")
|