Spaces:
Sleeping
Sleeping
Commit ·
ef4a402
1
Parent(s): 3450dc5
Fix /grader endpoint: revert to GraderRequest format, inject task_id into state
Browse filesThe 3450dc5 commit broke Task Validation by switching /grader to accept
{action, ground_truth} instead of {task_id, state, reward}. The external
HF validator calls /grader with the original format — causing 422 errors
and "Not enough tasks with graders" failure.
Also injects correct task_id into state when missing so graders always
return a valid score in [0.0, 1.0] regardless of what state the validator
sends.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
- server/app.py +24 -24
server/app.py
CHANGED
|
@@ -34,9 +34,10 @@ app = create_fastapi_app(
|
|
| 34 |
)
|
| 35 |
|
| 36 |
|
| 37 |
-
class
|
| 38 |
-
|
| 39 |
-
|
|
|
|
| 40 |
|
| 41 |
WEB_CSS = dedent(
|
| 42 |
"""\
|
|
@@ -468,8 +469,16 @@ def tasks() -> list[dict]:
|
|
| 468 |
|
| 469 |
|
| 470 |
@app.post("/grader", include_in_schema=False)
|
| 471 |
-
def grader(request:
|
| 472 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 473 |
"quiet-morning": grade_task_0,
|
| 474 |
"meeting-surgery": grade_task_1,
|
| 475 |
"delivery-triage": grade_task_2,
|
|
@@ -477,26 +486,17 @@ def grader(request: GradeRequest) -> JSONResponse:
|
|
| 477 |
"engineer_manager_task_1": grade_task_1,
|
| 478 |
"engineer_manager_task_2": grade_task_2,
|
| 479 |
}
|
| 480 |
-
|
| 481 |
-
reward = request.action.get("reward", 0.0)
|
| 482 |
-
state = {
|
| 483 |
-
"task_id": request.action.get("task_index"),
|
| 484 |
-
"task_name": request.action.get("task_name"),
|
| 485 |
-
"metadata": request.action.get("metadata", {}),
|
| 486 |
-
}
|
| 487 |
-
if state["task_id"] is None:
|
| 488 |
-
if task_id == "quiet-morning" or task_id == "engineer_manager_task_0":
|
| 489 |
-
state["task_id"] = 0
|
| 490 |
-
elif task_id == "meeting-surgery" or task_id == "engineer_manager_task_1":
|
| 491 |
-
state["task_id"] = 1
|
| 492 |
-
elif task_id == "delivery-triage" or task_id == "engineer_manager_task_2":
|
| 493 |
-
state["task_id"] = 2
|
| 494 |
-
|
| 495 |
-
grader_fn = graders.get(task_id)
|
| 496 |
if grader_fn is None:
|
| 497 |
-
return JSONResponse(
|
| 498 |
-
|
| 499 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 500 |
|
| 501 |
|
| 502 |
def run(host: str = "0.0.0.0", port: int = 8000) -> None:
|
|
|
|
| 34 |
)
|
| 35 |
|
| 36 |
|
| 37 |
+
class GraderRequest(BaseModel):
|
| 38 |
+
task_id: str
|
| 39 |
+
state: dict
|
| 40 |
+
reward: float
|
| 41 |
|
| 42 |
WEB_CSS = dedent(
|
| 43 |
"""\
|
|
|
|
| 469 |
|
| 470 |
|
| 471 |
@app.post("/grader", include_in_schema=False)
|
| 472 |
+
def grader(request: GraderRequest) -> JSONResponse:
|
| 473 |
+
task_index_map = {
|
| 474 |
+
"quiet-morning": 0,
|
| 475 |
+
"engineer_manager_task_0": 0,
|
| 476 |
+
"meeting-surgery": 1,
|
| 477 |
+
"engineer_manager_task_1": 1,
|
| 478 |
+
"delivery-triage": 2,
|
| 479 |
+
"engineer_manager_task_2": 2,
|
| 480 |
+
}
|
| 481 |
+
grader_fn_map = {
|
| 482 |
"quiet-morning": grade_task_0,
|
| 483 |
"meeting-surgery": grade_task_1,
|
| 484 |
"delivery-triage": grade_task_2,
|
|
|
|
| 486 |
"engineer_manager_task_1": grade_task_1,
|
| 487 |
"engineer_manager_task_2": grade_task_2,
|
| 488 |
}
|
| 489 |
+
grader_fn = grader_fn_map.get(request.task_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 490 |
if grader_fn is None:
|
| 491 |
+
return JSONResponse(
|
| 492 |
+
{"error": f"Unknown task_id: {request.task_id}", "score": 0.0, "passed": False},
|
| 493 |
+
status_code=400,
|
| 494 |
+
)
|
| 495 |
+
state = dict(request.state)
|
| 496 |
+
if "task_id" not in state or state["task_id"] is None:
|
| 497 |
+
state["task_id"] = task_index_map[request.task_id]
|
| 498 |
+
score = float(grader_fn(state, request.reward))
|
| 499 |
+
return JSONResponse({"task_id": request.task_id, "score": score, "passed": score >= 0.0, "reward": score})
|
| 500 |
|
| 501 |
|
| 502 |
def run(host: str = "0.0.0.0", port: int = 8000) -> None:
|