arnavmehta7 Claude Sonnet 4.6 commited on
Commit
ef4a402
·
1 Parent(s): 3450dc5

Fix /grader endpoint: revert to GraderRequest format, inject task_id into state

Browse files

The 3450dc5 commit broke Task Validation by switching /grader to accept
{action, ground_truth} instead of {task_id, state, reward}. The external
HF validator calls /grader with the original format — causing 422 errors
and "Not enough tasks with graders" failure.

Also injects correct task_id into state when missing so graders always
return a valid score in [0.0, 1.0] regardless of what state the validator
sends.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. server/app.py +24 -24
server/app.py CHANGED
@@ -34,9 +34,10 @@ app = create_fastapi_app(
34
  )
35
 
36
 
37
- class GradeRequest(BaseModel):
38
- action: dict
39
- ground_truth: dict | None = None
 
40
 
41
  WEB_CSS = dedent(
42
  """\
@@ -468,8 +469,16 @@ def tasks() -> list[dict]:
468
 
469
 
470
  @app.post("/grader", include_in_schema=False)
471
- def grader(request: GradeRequest) -> JSONResponse:
472
- graders = {
 
 
 
 
 
 
 
 
473
  "quiet-morning": grade_task_0,
474
  "meeting-surgery": grade_task_1,
475
  "delivery-triage": grade_task_2,
@@ -477,26 +486,17 @@ def grader(request: GradeRequest) -> JSONResponse:
477
  "engineer_manager_task_1": grade_task_1,
478
  "engineer_manager_task_2": grade_task_2,
479
  }
480
- task_id = request.action.get("task_id", "engineer_manager_task_0")
481
- reward = request.action.get("reward", 0.0)
482
- state = {
483
- "task_id": request.action.get("task_index"),
484
- "task_name": request.action.get("task_name"),
485
- "metadata": request.action.get("metadata", {}),
486
- }
487
- if state["task_id"] is None:
488
- if task_id == "quiet-morning" or task_id == "engineer_manager_task_0":
489
- state["task_id"] = 0
490
- elif task_id == "meeting-surgery" or task_id == "engineer_manager_task_1":
491
- state["task_id"] = 1
492
- elif task_id == "delivery-triage" or task_id == "engineer_manager_task_2":
493
- state["task_id"] = 2
494
-
495
- grader_fn = graders.get(task_id)
496
  if grader_fn is None:
497
- return JSONResponse({"score": 0.0}, status_code=200)
498
- score = float(grader_fn(state, reward))
499
- return JSONResponse({"score": score})
 
 
 
 
 
 
500
 
501
 
502
  def run(host: str = "0.0.0.0", port: int = 8000) -> None:
 
34
  )
35
 
36
 
37
+ class GraderRequest(BaseModel):
38
+ task_id: str
39
+ state: dict
40
+ reward: float
41
 
42
  WEB_CSS = dedent(
43
  """\
 
469
 
470
 
471
  @app.post("/grader", include_in_schema=False)
472
+ def grader(request: GraderRequest) -> JSONResponse:
473
+ task_index_map = {
474
+ "quiet-morning": 0,
475
+ "engineer_manager_task_0": 0,
476
+ "meeting-surgery": 1,
477
+ "engineer_manager_task_1": 1,
478
+ "delivery-triage": 2,
479
+ "engineer_manager_task_2": 2,
480
+ }
481
+ grader_fn_map = {
482
  "quiet-morning": grade_task_0,
483
  "meeting-surgery": grade_task_1,
484
  "delivery-triage": grade_task_2,
 
486
  "engineer_manager_task_1": grade_task_1,
487
  "engineer_manager_task_2": grade_task_2,
488
  }
489
+ grader_fn = grader_fn_map.get(request.task_id)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
490
  if grader_fn is None:
491
+ return JSONResponse(
492
+ {"error": f"Unknown task_id: {request.task_id}", "score": 0.0, "passed": False},
493
+ status_code=400,
494
+ )
495
+ state = dict(request.state)
496
+ if "task_id" not in state or state["task_id"] is None:
497
+ state["task_id"] = task_index_map[request.task_id]
498
+ score = float(grader_fn(state, request.reward))
499
+ return JSONResponse({"task_id": request.task_id, "score": score, "passed": score >= 0.0, "reward": score})
500
 
501
 
502
  def run(host: str = "0.0.0.0", port: int = 8000) -> None: