Spaces:
Sleeping
Sleeping
File size: 890 Bytes
46fc2a0 c8e80a5 46fc2a0 c8e80a5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 | """Grader for server_error task: 500 Internal Server Error."""
def grade(trajectory) -> float:
"""Grade the server_error task based on agent trajectory.
Args:
trajectory: List of (action, observation, reward) tuples from the episode.
Returns:
Score between 0 and 1.
"""
if not trajectory:
return 0.0
# Check if agent escalated to handle the server error
actions = [step[0] if isinstance(step, (list, tuple)) else step.get("action", "") for step in trajectory]
correct_action_used = "escalate" in actions
resolved = any(
(step[2] > 0 if isinstance(step, (list, tuple)) else step.get("reward", 0) > 0)
for step in trajectory
)
if correct_action_used and resolved:
return 0.95
elif correct_action_used:
return 0.6
elif resolved:
return 0.4
else:
return 0.05
|