Deepikachintamreddy commited on
Commit ·
b53b055
1
Parent(s): 4662d81
fix: resolve merge conflict in grader_api.py
Browse files- openenv.yaml +1 -1
- server/env.py +1 -1
- server/graders/grader_api.py +31 -27
- verify_graders.py +51 -0
openenv.yaml
CHANGED
|
@@ -59,4 +59,4 @@ tasks:
|
|
| 59 |
difficulty: very_hard
|
| 60 |
num_bugs: 6
|
| 61 |
grader: server.graders.grader_api:grade_task7
|
| 62 |
-
has_grader: true
|
|
|
|
| 59 |
difficulty: very_hard
|
| 60 |
num_bugs: 6
|
| 61 |
grader: server.graders.grader_api:grade_task7
|
| 62 |
+
has_grader: true
|
server/env.py
CHANGED
|
@@ -363,4 +363,4 @@ with gr.Blocks(title="ConfigDebugEnv", theme=gr.themes.Soft()) as demo:
|
|
| 363 |
outputs=[state_display],
|
| 364 |
)
|
| 365 |
|
| 366 |
-
app = gr.mount_gradio_app(app, demo, path="/")
|
|
|
|
| 363 |
outputs=[state_display],
|
| 364 |
)
|
| 365 |
|
| 366 |
+
app = gr.mount_gradio_app(app, demo, path="/")
|
server/graders/grader_api.py
CHANGED
|
@@ -1,13 +1,6 @@
|
|
| 1 |
"""
|
| 2 |
-
grader_api.py
|
| 3 |
-
|
| 4 |
-
The hackathon validator imports graders via the paths in openenv.yaml
|
| 5 |
-
and expects each grader to return a SINGLE FLOAT score in (0, 1).
|
| 6 |
-
|
| 7 |
-
NOT a tuple. NOT a dict. Just a float.
|
| 8 |
-
|
| 9 |
-
Internal code (task_registry) imports from the raw grader files
|
| 10 |
-
which return (reward, error_msg, bugs_fixed) tuples.
|
| 11 |
"""
|
| 12 |
|
| 13 |
from server.graders.json_grader import grade_task1 as _raw_grade_task1
|
|
@@ -19,25 +12,36 @@ from server.graders.github_actions_grader import grade_task6 as _raw_grade_task6
|
|
| 19 |
from server.graders.nginx_grader import grade_task7 as _raw_grade_task7
|
| 20 |
|
| 21 |
|
| 22 |
-
def
|
| 23 |
-
"""Wrap a grader: call it, extract reward float, clamp to (0.01, 0.99)."""
|
| 24 |
def wrapper(submitted_config: str) -> float:
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
|
|
|
|
|
|
|
|
|
| 34 |
return wrapper
|
| 35 |
|
| 36 |
|
| 37 |
-
grade_task1 =
|
| 38 |
-
grade_task2 =
|
| 39 |
-
grade_task3 =
|
| 40 |
-
grade_task4 =
|
| 41 |
-
grade_task5 =
|
| 42 |
-
grade_task6 =
|
| 43 |
-
grade_task7 =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""
|
| 2 |
+
grader_api.py - Normalized grader wrappers for OpenEnv validator.
|
| 3 |
+
Returns ONLY a float strictly between 0 and 1.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
"""
|
| 5 |
|
| 6 |
from server.graders.json_grader import grade_task1 as _raw_grade_task1
|
|
|
|
| 12 |
from server.graders.nginx_grader import grade_task7 as _raw_grade_task7
|
| 13 |
|
| 14 |
|
| 15 |
+
def _make_float_grader(raw_fn):
|
|
|
|
| 16 |
def wrapper(submitted_config: str) -> float:
|
| 17 |
+
try:
|
| 18 |
+
result = raw_fn(submitted_config)
|
| 19 |
+
if isinstance(result, (list, tuple)):
|
| 20 |
+
reward = float(result[0])
|
| 21 |
+
else:
|
| 22 |
+
reward = float(result)
|
| 23 |
+
except Exception:
|
| 24 |
+
reward = 0.01
|
| 25 |
+
reward = max(0.01, min(0.99, reward))
|
| 26 |
+
return reward
|
| 27 |
+
wrapper.__name__ = raw_fn.__name__
|
| 28 |
+
wrapper.__qualname__ = raw_fn.__qualname__
|
| 29 |
return wrapper
|
| 30 |
|
| 31 |
|
| 32 |
+
grade_task1 = _make_float_grader(_raw_grade_task1)
|
| 33 |
+
grade_task2 = _make_float_grader(_raw_grade_task2)
|
| 34 |
+
grade_task3 = _make_float_grader(_raw_grade_task3)
|
| 35 |
+
grade_task4 = _make_float_grader(_raw_grade_task4)
|
| 36 |
+
grade_task5 = _make_float_grader(_raw_grade_task5)
|
| 37 |
+
grade_task6 = _make_float_grader(_raw_grade_task6)
|
| 38 |
+
grade_task7 = _make_float_grader(_raw_grade_task7)
|
| 39 |
+
|
| 40 |
+
# Aliases: app.py generates names like grade_task1_json, grade_task2_yaml etc.
|
| 41 |
+
grade_task1_json = grade_task1
|
| 42 |
+
grade_task2_yaml = grade_task2
|
| 43 |
+
grade_task3_dockerfile = grade_task3
|
| 44 |
+
grade_task4_compose = grade_task4
|
| 45 |
+
grade_task5_k8s = grade_task5
|
| 46 |
+
grade_task6_github_actions = grade_task6
|
| 47 |
+
grade_task7_nginx = grade_task7
|
verify_graders.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import sys, os
|
| 2 |
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
| 3 |
+
|
| 4 |
+
def main():
|
| 5 |
+
print("=" * 60)
|
| 6 |
+
print("GRADER VERIFICATION")
|
| 7 |
+
print("=" * 60)
|
| 8 |
+
|
| 9 |
+
print("\n[TEST 1] Importing grader_api functions...")
|
| 10 |
+
try:
|
| 11 |
+
from server.graders.grader_api import (
|
| 12 |
+
grade_task1, grade_task2, grade_task3,
|
| 13 |
+
grade_task4, grade_task5, grade_task6, grade_task7,
|
| 14 |
+
)
|
| 15 |
+
print(" OK - All 7 graders imported")
|
| 16 |
+
except ImportError as e:
|
| 17 |
+
print(f" FAIL - {e}")
|
| 18 |
+
sys.exit(1)
|
| 19 |
+
|
| 20 |
+
print("\n[TEST 2] Calling graders, checking type + range...")
|
| 21 |
+
graders = [
|
| 22 |
+
("task1", grade_task1), ("task2", grade_task2),
|
| 23 |
+
("task3", grade_task3), ("task4", grade_task4),
|
| 24 |
+
("task5", grade_task5), ("task6", grade_task6),
|
| 25 |
+
("task7", grade_task7),
|
| 26 |
+
]
|
| 27 |
+
all_ok = True
|
| 28 |
+
for name, fn in graders:
|
| 29 |
+
try:
|
| 30 |
+
result = fn("{}")
|
| 31 |
+
if not isinstance(result, float):
|
| 32 |
+
print(f" FAIL - {name}: returned {type(result).__name__}, need float")
|
| 33 |
+
all_ok = False
|
| 34 |
+
elif result <= 0.0 or result >= 1.0:
|
| 35 |
+
print(f" FAIL - {name}: score={result} NOT in (0,1)")
|
| 36 |
+
all_ok = False
|
| 37 |
+
else:
|
| 38 |
+
print(f" OK - {name}: score={result}")
|
| 39 |
+
except Exception as e:
|
| 40 |
+
print(f" FAIL - {name}: {e}")
|
| 41 |
+
all_ok = False
|
| 42 |
+
|
| 43 |
+
print("\n" + "=" * 60)
|
| 44 |
+
if all_ok:
|
| 45 |
+
print("ALL PASSED - Push and submit!")
|
| 46 |
+
else:
|
| 47 |
+
print("FAILED - Fix before submitting!")
|
| 48 |
+
print("=" * 60)
|
| 49 |
+
|
| 50 |
+
if __name__ == "__main__":
|
| 51 |
+
main()
|