tasks/timeout/grader.py

"""Grader for timeout task: 408 Request Timeout."""


def grade(trajectory) -> float:
    """Grade the timeout task based on agent trajectory.

    Args:
        trajectory: List of (action, observation, reward) tuples from the episode.

    Returns:
        Score between 0 and 1.
    """
    if not trajectory:
        return 0.0

    # Check if agent used wait_retry to fix the 408 timeout error
    actions = [step[0] if isinstance(step, (list, tuple)) else step.get("action", "") for step in trajectory]

    correct_action_used = "wait_retry" in actions
    resolved = any(
        (step[2] > 0 if isinstance(step, (list, tuple)) else step.get("reward", 0) > 0)
        for step in trajectory
    )

    if correct_action_used and resolved:
        return 0.95
    elif correct_action_used:
        return 0.6
    elif resolved:
        return 0.4
    else:
        return 0.05