Spaces:
Sleeping
Sleeping
Commit Β·
95b11e6
1
Parent(s): 49552e4
done
Browse files- env/graders.py +6 -6
env/graders.py
CHANGED
|
@@ -132,7 +132,7 @@ def grade_easy(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
|
|
| 132 |
"""
|
| 133 |
# Edge case: null or malformed action
|
| 134 |
if action is None or action.payload is None:
|
| 135 |
-
return 0.
|
| 136 |
|
| 137 |
payload = action.payload
|
| 138 |
score = 0.0
|
|
@@ -197,7 +197,7 @@ def grade_easy(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
|
|
| 197 |
# ββ 6. Hint penalty ββββββββββββββββββββββββββββββββββββββββββ
|
| 198 |
# Hint penalty is applied in reward.py, not here
|
| 199 |
|
| 200 |
-
final_score = round(min(score, 0.999), 4)
|
| 201 |
feedback = " ".join(feedback_parts) if feedback_parts else "No valid response provided."
|
| 202 |
return final_score, breakdown, feedback
|
| 203 |
|
|
@@ -288,7 +288,7 @@ def grade_medium(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
|
|
| 288 |
else:
|
| 289 |
breakdown["impact_analysis"] = 0.0
|
| 290 |
|
| 291 |
-
final_score = round(min(score, 0.999), 4)
|
| 292 |
feedback = " ".join(feedback_parts) if feedback_parts else "No valid response provided."
|
| 293 |
return final_score, breakdown, feedback
|
| 294 |
|
|
@@ -399,7 +399,7 @@ def grade_hard(action: Action, ground_truth: dict) -> tuple[float, dict, str]:
|
|
| 399 |
|
| 400 |
# Hard cap: frontier model should score ~0.10-0.20
|
| 401 |
# We do NOT artificially cap β the rubric naturally produces low scores
|
| 402 |
-
final_score = round(min(score, 0.999), 4)
|
| 403 |
feedback = " ".join(feedback_parts) if feedback_parts else "Performance issue not identified."
|
| 404 |
return final_score, breakdown, feedback
|
| 405 |
|
|
@@ -421,7 +421,7 @@ def grade(action: Action, task_id: str) -> tuple[float, dict, str]:
|
|
| 421 |
# Edge case: unknown task
|
| 422 |
ground_truth = task_manager.get_ground_truth(task_id)
|
| 423 |
if ground_truth is None:
|
| 424 |
-
return 0.
|
| 425 |
|
| 426 |
# Dispatch by difficulty
|
| 427 |
difficulty = ground_truth.get("id", "").split("_")[0]
|
|
@@ -437,4 +437,4 @@ def grade(action: Action, task_id: str) -> tuple[float, dict, str]:
|
|
| 437 |
return 0.0, {"error": "unknown_difficulty"}, f"Unknown difficulty: {difficulty}"
|
| 438 |
except Exception as e:
|
| 439 |
# Never crash β return 0.0 with error info
|
| 440 |
-
return 0.
|
|
|
|
| 132 |
"""
|
| 133 |
# Edge case: null or malformed action
|
| 134 |
if action is None or action.payload is None:
|
| 135 |
+
return 0.001, {"error": "null_action"}, "No action provided."
|
| 136 |
|
| 137 |
payload = action.payload
|
| 138 |
score = 0.0
|
|
|
|
| 197 |
# ββ 6. Hint penalty ββββββββββββββββββββββββββββββββββββββββββ
|
| 198 |
# Hint penalty is applied in reward.py, not here
|
| 199 |
|
| 200 |
+
final_score = round(max(min(score, 0.999), 0.001), 4)
|
| 201 |
feedback = " ".join(feedback_parts) if feedback_parts else "No valid response provided."
|
| 202 |
return final_score, breakdown, feedback
|
| 203 |
|
|
|
|
| 288 |
else:
|
| 289 |
breakdown["impact_analysis"] = 0.0
|
| 290 |
|
| 291 |
+
final_score = round(max(min(score, 0.999), 0.001), 4)
|
| 292 |
feedback = " ".join(feedback_parts) if feedback_parts else "No valid response provided."
|
| 293 |
return final_score, breakdown, feedback
|
| 294 |
|
|
|
|
| 399 |
|
| 400 |
# Hard cap: frontier model should score ~0.10-0.20
|
| 401 |
# We do NOT artificially cap β the rubric naturally produces low scores
|
| 402 |
+
final_score = round(max(min(score, 0.999), 0.001), 4)
|
| 403 |
feedback = " ".join(feedback_parts) if feedback_parts else "Performance issue not identified."
|
| 404 |
return final_score, breakdown, feedback
|
| 405 |
|
|
|
|
| 421 |
# Edge case: unknown task
|
| 422 |
ground_truth = task_manager.get_ground_truth(task_id)
|
| 423 |
if ground_truth is None:
|
| 424 |
+
return 0.001, {"error": "unknown_task"}, f"Task '{task_id}' not found."
|
| 425 |
|
| 426 |
# Dispatch by difficulty
|
| 427 |
difficulty = ground_truth.get("id", "").split("_")[0]
|
|
|
|
| 437 |
return 0.0, {"error": "unknown_difficulty"}, f"Unknown difficulty: {difficulty}"
|
| 438 |
except Exception as e:
|
| 439 |
# Never crash β return 0.0 with error info
|
| 440 |
+
return 0.001, {"error": str(e)}, f"Grader error: {str(e)}"
|