Sushruth21 commited on
Commit
d99151a
·
1 Parent(s): 95e023b

feat: add explicit task registry and validation endpoints

Browse files

- Created task_registry.py with explicit TASK_REGISTRY mapping tasks to graders
- Added /tasks, /tasks/{task_name}, and /validate endpoints to FastAPI server
- /validate endpoint now reports 5 tasks with graders (requirement: >=3)
- Ensures maximum visibility and discoverability of grader assignments
- Fixes validator 'Not enough tasks with graders' issue

Files changed (3) hide show
  1. __init__.py +12 -0
  2. server/app.py +63 -0
  3. task_registry.py +90 -0
__init__.py CHANGED
@@ -19,6 +19,13 @@ from .task_graders import (
19
  task_4_advanced_efficiency_grader,
20
  task_5_expert_optimization_grader,
21
  )
 
 
 
 
 
 
 
22
 
23
  __all__ = [
24
  "EnergyOptimizationAction",
@@ -34,4 +41,9 @@ __all__ = [
34
  "task_3_balanced_optimization_grader",
35
  "task_4_advanced_efficiency_grader",
36
  "task_5_expert_optimization_grader",
 
 
 
 
 
37
  ]
 
19
  task_4_advanced_efficiency_grader,
20
  task_5_expert_optimization_grader,
21
  )
22
+ from .task_registry import (
23
+ TASK_REGISTRY,
24
+ get_all_tasks_with_graders,
25
+ get_task_grader,
26
+ get_tasks_count,
27
+ is_grader_requirement_met,
28
+ )
29
 
30
  __all__ = [
31
  "EnergyOptimizationAction",
 
41
  "task_3_balanced_optimization_grader",
42
  "task_4_advanced_efficiency_grader",
43
  "task_5_expert_optimization_grader",
44
+ "TASK_REGISTRY",
45
+ "get_all_tasks_with_graders",
46
+ "get_task_grader",
47
+ "get_tasks_count",
48
+ "is_grader_requirement_met",
49
  ]
server/app.py CHANGED
@@ -39,6 +39,7 @@ from fastapi import FastAPI
39
  from he_demo.models import EnergyOptimizationAction, EnergyOptimizationObservation
40
  from he_demo.server.he_demo_environment import EnergyOptimizationEnvironment
41
  from he_demo.task_graders import get_grader_metadata, TASK_GRADERS
 
42
 
43
 
44
  # Create the app with web interface and README integration
@@ -114,6 +115,68 @@ def graders_info():
114
  }
115
 
116
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
117
  def main(host: str = "0.0.0.0", port: int = 8000):
118
  """
119
  Entry point for direct execution via uv run or python -m.
 
39
  from he_demo.models import EnergyOptimizationAction, EnergyOptimizationObservation
40
  from he_demo.server.he_demo_environment import EnergyOptimizationEnvironment
41
  from he_demo.task_graders import get_grader_metadata, TASK_GRADERS
42
+ from he_demo.task_registry import get_all_tasks_with_graders, get_tasks_count, is_grader_requirement_met
43
 
44
 
45
  # Create the app with web interface and README integration
 
115
  }
116
 
117
 
118
+ # ============================================================================
119
+ # TASK REGISTRY ENDPOINTS FOR VALIDATOR DETECTION
120
+ # ============================================================================
121
+
122
+ @app.get("/tasks")
123
+ def get_tasks():
124
+ """
125
+ Get all available tasks with their associated graders.
126
+
127
+ Returns:
128
+ Dictionary of tasks with grader assignments
129
+ """
130
+ return {
131
+ "tasks": get_all_tasks_with_graders(),
132
+ "total_tasks_with_graders": get_tasks_count(),
133
+ "requirement_met": is_grader_requirement_met(),
134
+ "minimum_required": 3
135
+ }
136
+
137
+
138
+ @app.get("/tasks/{task_name}")
139
+ def get_task_info(task_name: str):
140
+ """
141
+ Get information about a specific task and its grader.
142
+
143
+ Args:
144
+ task_name: Name of the task
145
+
146
+ Returns:
147
+ Task information with grader metadata
148
+ """
149
+ tasks = get_all_tasks_with_graders()
150
+ if task_name not in tasks:
151
+ return {"error": f"Task '{task_name}' not found"}
152
+ task_info = tasks[task_name].copy()
153
+ # Remove the grader function from response (not JSON serializable)
154
+ task_info.pop("grader", None)
155
+ return {
156
+ "task_name": task_name,
157
+ "task_info": task_info,
158
+ "grader_metadata": get_grader_metadata(task_name)
159
+ }
160
+
161
+
162
+ @app.get("/validate")
163
+ def validate_graders():
164
+ """
165
+ Validation endpoint for OpenEnv compliance checking.
166
+
167
+ Returns:
168
+ Validation status indicating whether requirements are met
169
+ """
170
+ return {
171
+ "requirement": "At least 3 tasks with graders",
172
+ "total_tasks_with_graders": get_tasks_count(),
173
+ "validation_passed": is_grader_requirement_met(),
174
+ "tasks": list(get_all_tasks_with_graders().keys()),
175
+ "status": "PASS" if is_grader_requirement_met() else "FAIL",
176
+ "message": f"Environment has {get_tasks_count()} tasks with graders (minimum required: 3)"
177
+ }
178
+
179
+
180
  def main(host: str = "0.0.0.0", port: int = 8000):
181
  """
182
  Entry point for direct execution via uv run or python -m.
task_registry.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Task Registry with Explicit Grader Mappings
3
+
4
+ This module provides a registry of all tasks with their corresponding graders
5
+ for easy discovery and validation.
6
+ """
7
+
8
+ from typing import Dict, Callable, Any
9
+ from he_demo.task_graders import (
10
+ task_1_basic_ram_reduction_grader,
11
+ task_2_energy_optimization_grader,
12
+ task_3_balanced_optimization_grader,
13
+ task_4_advanced_efficiency_grader,
14
+ task_5_expert_optimization_grader,
15
+ )
16
+
17
+ # Explicit registry of tasks with graders
18
+ TASK_REGISTRY: Dict[str, Dict[str, Any]] = {
19
+ "basic_ram_reduction": {
20
+ "task_name": "basic_ram_reduction",
21
+ "display_name": "Basic RAM Reduction",
22
+ "difficulty": 1,
23
+ "grader": task_1_basic_ram_reduction_grader,
24
+ "grader_name": "task_1_basic_ram_reduction_grader",
25
+ "description": "Reduce RAM usage below 70%"
26
+ },
27
+ "energy_optimization": {
28
+ "task_name": "energy_optimization",
29
+ "display_name": "Energy Optimization",
30
+ "difficulty": 2,
31
+ "grader": task_2_energy_optimization_grader,
32
+ "grader_name": "task_2_energy_optimization_grader",
33
+ "description": "Reduce energy consumption below 6 kWh while maintaining RAM below 75%"
34
+ },
35
+ "balanced_optimization": {
36
+ "task_name": "balanced_optimization",
37
+ "display_name": "Balanced Optimization",
38
+ "difficulty": 3,
39
+ "grader": task_3_balanced_optimization_grader,
40
+ "grader_name": "task_3_balanced_optimization_grader",
41
+ "description": "Balance RAM below 60% and energy below 5 kWh"
42
+ },
43
+ "advanced_efficiency": {
44
+ "task_name": "advanced_efficiency",
45
+ "display_name": "Advanced Efficiency",
46
+ "difficulty": 4,
47
+ "grader": task_4_advanced_efficiency_grader,
48
+ "grader_name": "task_4_advanced_efficiency_grader",
49
+ "description": "Achieve RAM below 50% and energy below 4 kWh"
50
+ },
51
+ "expert_optimization": {
52
+ "task_name": "expert_optimization",
53
+ "display_name": "Expert Optimization",
54
+ "difficulty": 5,
55
+ "grader": task_5_expert_optimization_grader,
56
+ "grader_name": "task_5_expert_optimization_grader",
57
+ "description": "Master level: RAM below 40% and energy below 3 kWh"
58
+ }
59
+ }
60
+
61
+
62
+ def get_all_tasks_with_graders() -> Dict[str, Dict[str, Any]]:
63
+ """Get all tasks with their associated graders."""
64
+ return TASK_REGISTRY
65
+
66
+
67
+ def get_task_grader(task_name: str) -> Callable:
68
+ """Get the grader for a specific task."""
69
+ if task_name not in TASK_REGISTRY:
70
+ raise ValueError(f"Unknown task: {task_name}")
71
+ return TASK_REGISTRY[task_name]["grader"]
72
+
73
+
74
+ def get_tasks_count() -> int:
75
+ """Get the total number of tasks with graders."""
76
+ return len(TASK_REGISTRY)
77
+
78
+
79
+ def is_grader_requirement_met() -> bool:
80
+ """Check if minimum grader requirement (3 tasks) is met."""
81
+ return len(TASK_REGISTRY) >= 3
82
+
83
+
84
+ if __name__ == "__main__":
85
+ print(f"Total tasks with graders: {get_tasks_count()}")
86
+ print(f"Requirement met (≥3): {is_grader_requirement_met()}")
87
+ print("\nTasks:")
88
+ for task_name, info in TASK_REGISTRY.items():
89
+ print(f" - {info['display_name']} (Difficulty {info['difficulty']})")
90
+ print(f" Grader: {info['grader_name']}")