""" Deterministic final grader. Returns a score in [0.0, 1.0] based on the terminal state. """ from .models import EnvState def grade(state: EnvState) -> float: """ Deterministic final score for the environment state. Score = weighted combination: 60% - goal completion rate (weighted by goal priority) Goals are the explicit objectives defined per task. They carry the highest weight because completing the right goal (e.g. saving a $18M term sheet) matters far more than handling any random email. 25% - priority-weighted email handling rate Measures breadth of coverage across the inbox. Ensures the agent doesn't only cherry-pick goal emails while ignoring other important items. 15% - efficiency bonus (only awarded if all high-priority goals are done) Rewards faster completion, but only after critical goals are met - efficiency never trumps correctness. Returns: float in [0.0, 1.0] """ goals = state.goals inbox = state.inbox # --- 60%: Goal completion (priority-weighted) --- goal_score = 0.0 if goals: total_priority = sum(g.priority for g in goals) completed_priority = sum(g.priority for g in goals if g.completed) goal_score = completed_priority / total_priority if total_priority > 0 else 0.0 # --- 25%: Priority-weighted email handling --- email_score = 0.0 if inbox: total_email_priority = sum(e.priority for e in inbox) handled_email_priority = sum(e.priority for e in inbox if e.handled) email_score = handled_email_priority / total_email_priority if total_email_priority > 0 else 0.0 # --- 15%: Efficiency bonus --- efficiency_score = 0.0 high_priority_goals = [g for g in goals if g.priority >= 4] all_high_priority_done = all(g.completed for g in high_priority_goals) if all_high_priority_done and high_priority_goals: steps_used = state.current_step steps_ratio = steps_used / state.max_steps if state.max_steps > 0 else 1.0 efficiency_score = max(0.0, 1.0 - steps_ratio) # --- Weighted combination --- final_score = ( 0.60 * goal_score + 0.25 * email_score + 0.15 * efficiency_score ) return max(0.0, min(1.0, final_score))