Spaces:
Sleeping
Sleeping
File size: 2,344 Bytes
1bb18a0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """
Deterministic final grader.
Returns a score in [0.0, 1.0] based on the terminal state.
"""
from .models import EnvState
def grade(state: EnvState) -> float:
"""
Deterministic final score for the environment state.
Score = weighted combination:
60% - goal completion rate (weighted by goal priority)
Goals are the explicit objectives defined per task. They carry the
highest weight because completing the right goal (e.g. saving a $18M
term sheet) matters far more than handling any random email.
25% - priority-weighted email handling rate
Measures breadth of coverage across the inbox. Ensures the agent
doesn't only cherry-pick goal emails while ignoring other important items.
15% - efficiency bonus (only awarded if all high-priority goals are done)
Rewards faster completion, but only after critical goals are met -
efficiency never trumps correctness.
Returns: float in [0.0, 1.0]
"""
goals = state.goals
inbox = state.inbox
# --- 60%: Goal completion (priority-weighted) ---
goal_score = 0.0
if goals:
total_priority = sum(g.priority for g in goals)
completed_priority = sum(g.priority for g in goals if g.completed)
goal_score = completed_priority / total_priority if total_priority > 0 else 0.0
# --- 25%: Priority-weighted email handling ---
email_score = 0.0
if inbox:
total_email_priority = sum(e.priority for e in inbox)
handled_email_priority = sum(e.priority for e in inbox if e.handled)
email_score = handled_email_priority / total_email_priority if total_email_priority > 0 else 0.0
# --- 15%: Efficiency bonus ---
efficiency_score = 0.0
high_priority_goals = [g for g in goals if g.priority >= 4]
all_high_priority_done = all(g.completed for g in high_priority_goals)
if all_high_priority_done and high_priority_goals:
steps_used = state.current_step
steps_ratio = steps_used / state.max_steps if state.max_steps > 0 else 1.0
efficiency_score = max(0.0, 1.0 - steps_ratio)
# --- Weighted combination ---
final_score = (
0.60 * goal_score +
0.25 * email_score +
0.15 * efficiency_score
)
return max(0.0, min(1.0, final_score))
|