""" Medium Grader — Scores multi-step issue resolution responses. """ from typing import Tuple, Optional class MediumGrader: """ Scoring logic for Medium (Multi-Step) Task. Formula: score = step_detection_score * completeness_multiplier * tone_multiplier Step Detection: Each required step has associated keywords. If the agent's reply matches those keywords → step is identified. Completeness: Partial answers (missing some required info for the step) get partial credit. Tone: Professional, empathetic tone adds a small bonus. """ # Keywords that signal each step type STEP_KEYWORDS = { # Medium scenario step types → what words trigger them "acknowledge_frustration": ["understand", "frustrat", "sorry to hear", "apologize", "that must", "i can see"], "acknowledge_issue": ["understand", "sorry", "i can see", "must be", "apologize", "frustrat"], "collect_account_info": ["email", "account", "order number", "name", "id", "could you provide", "can i get", "please share"], "collect_order_info": ["order number", "order id", "reference", "can i get", "could you share"], "investigate": ["look into", "check", "investigate", "pull up", "look at", "review your account"], "check_basic_steps": ["browser", "cache", "clear", "try", "incognito", "different browser", "cookies"], "offer_password_reset": ["reset", "password", "link", "send you", "email you", "forgot password"], "confirm_resolution": ["resolved", "able to log", "working now", "fixed", "sorted", "everything ok", "is that working"], "resolve_or_escalate": ["refund", "credit", "resolve", "fix", "escalate", "team will", "processed"], "apologize_sincerely": ["sorry", "apologize", "sincerely apologize", "deeply sorry", "truly sorry"], "confirm_item_details": ["ordered", "confirm", "blue jacket", "correct item", "you ordered", "size"], "arrange_replacement": ["replacement", "send the correct", "return", "new", "reship", "express", "free return"], } POLITE_PHRASES = [ "happy to help", "certainly", "of course", "absolutely", "please", "thank you", "glad", "i understand", "i appreciate", "let me help", "my pleasure", ] BAD_PHRASES = [ "not my problem", "policy says", "can't do anything", "nothing i can do", "you should have", "your fault", "read the faq", ] def grade( self, action: str, scenario: dict, history: list, steps_completed: list, ) -> Tuple[float, dict, Optional[str]]: """ Grade the agent's response for a medium-task step. Returns: (score: float, grader_info: dict, step_identified: str or None) """ action_lower = action.lower() required_steps = scenario.get("required_steps", []) # 1. Detect which step the agent is performing step_identified = None step_match_score = 0.0 for step in required_steps: if step in steps_completed: continue # Already done, skip step_keywords = self.STEP_KEYWORDS.get(step, []) if not step_keywords: continue matches = sum(1 for kw in step_keywords if kw in action_lower) match_ratio = matches / len(step_keywords) if match_ratio > step_match_score and matches >= 1: step_match_score = match_ratio step_identified = step # 2. Completeness score (how thoroughly the step is addressed) completeness = min(1.0, step_match_score * 1.5) if step_identified else 0.3 # 3. Tone multiplier has_polite = any(p in action_lower for p in self.POLITE_PHRASES) has_bad = any(p in action_lower for p in self.BAD_PHRASES) if has_bad: tone_mult = 0.6 tone = "bad" elif has_polite: tone_mult = 1.1 tone = "polite" else: tone_mult = 1.0 tone = "neutral" # Final score if step_identified: score = min(1.0, completeness * tone_mult) else: # No useful step found — partial score for being polite score = 0.2 * tone_mult if has_polite else 0.1 grader_info = { "step_identified": step_identified, "step_match_score": round(step_match_score, 3), "completeness": round(completeness, 3), "tone": tone, "already_done": steps_completed, } return round(score, 3), grader_info, step_identified