Spaces:
Sleeping
Sleeping
| """ | |
| Medium Grader — Scores multi-step issue resolution responses. | |
| """ | |
| from typing import Tuple, Optional | |
| class MediumGrader: | |
| """ | |
| Scoring logic for Medium (Multi-Step) Task. | |
| Formula: | |
| score = step_detection_score * completeness_multiplier * tone_multiplier | |
| Step Detection: | |
| Each required step has associated keywords. | |
| If the agent's reply matches those keywords → step is identified. | |
| Completeness: | |
| Partial answers (missing some required info for the step) get partial credit. | |
| Tone: | |
| Professional, empathetic tone adds a small bonus. | |
| """ | |
| # Keywords that signal each step type | |
| STEP_KEYWORDS = { | |
| # Medium scenario step types → what words trigger them | |
| "acknowledge_frustration": ["understand", "frustrat", "sorry to hear", "apologize", "that must", "i can see"], | |
| "acknowledge_issue": ["understand", "sorry", "i can see", "must be", "apologize", "frustrat"], | |
| "collect_account_info": ["email", "account", "order number", "name", "id", "could you provide", "can i get", "please share"], | |
| "collect_order_info": ["order number", "order id", "reference", "can i get", "could you share"], | |
| "investigate": ["look into", "check", "investigate", "pull up", "look at", "review your account"], | |
| "check_basic_steps": ["browser", "cache", "clear", "try", "incognito", "different browser", "cookies"], | |
| "offer_password_reset": ["reset", "password", "link", "send you", "email you", "forgot password"], | |
| "confirm_resolution": ["resolved", "able to log", "working now", "fixed", "sorted", "everything ok", "is that working"], | |
| "resolve_or_escalate": ["refund", "credit", "resolve", "fix", "escalate", "team will", "processed"], | |
| "apologize_sincerely": ["sorry", "apologize", "sincerely apologize", "deeply sorry", "truly sorry"], | |
| "confirm_item_details": ["ordered", "confirm", "blue jacket", "correct item", "you ordered", "size"], | |
| "arrange_replacement": ["replacement", "send the correct", "return", "new", "reship", "express", "free return"], | |
| } | |
| POLITE_PHRASES = [ | |
| "happy to help", "certainly", "of course", "absolutely", | |
| "please", "thank you", "glad", "i understand", "i appreciate", | |
| "let me help", "my pleasure", | |
| ] | |
| BAD_PHRASES = [ | |
| "not my problem", "policy says", "can't do anything", "nothing i can do", | |
| "you should have", "your fault", "read the faq", | |
| ] | |
| def grade( | |
| self, | |
| action: str, | |
| scenario: dict, | |
| history: list, | |
| steps_completed: list, | |
| ) -> Tuple[float, dict, Optional[str]]: | |
| """ | |
| Grade the agent's response for a medium-task step. | |
| Returns: | |
| (score: float, grader_info: dict, step_identified: str or None) | |
| """ | |
| action_lower = action.lower() | |
| required_steps = scenario.get("required_steps", []) | |
| # 1. Detect which step the agent is performing | |
| step_identified = None | |
| step_match_score = 0.0 | |
| for step in required_steps: | |
| if step in steps_completed: | |
| continue # Already done, skip | |
| step_keywords = self.STEP_KEYWORDS.get(step, []) | |
| if not step_keywords: | |
| continue | |
| matches = sum(1 for kw in step_keywords if kw in action_lower) | |
| match_ratio = matches / len(step_keywords) | |
| if match_ratio > step_match_score and matches >= 1: | |
| step_match_score = match_ratio | |
| step_identified = step | |
| # 2. Completeness score (how thoroughly the step is addressed) | |
| completeness = min(1.0, step_match_score * 1.5) if step_identified else 0.3 | |
| # 3. Tone multiplier | |
| has_polite = any(p in action_lower for p in self.POLITE_PHRASES) | |
| has_bad = any(p in action_lower for p in self.BAD_PHRASES) | |
| if has_bad: | |
| tone_mult = 0.6 | |
| tone = "bad" | |
| elif has_polite: | |
| tone_mult = 1.1 | |
| tone = "polite" | |
| else: | |
| tone_mult = 1.0 | |
| tone = "neutral" | |
| # Final score | |
| if step_identified: | |
| score = min(1.0, completeness * tone_mult) | |
| else: | |
| # No useful step found — partial score for being polite | |
| score = 0.2 * tone_mult if has_polite else 0.1 | |
| grader_info = { | |
| "step_identified": step_identified, | |
| "step_match_score": round(step_match_score, 3), | |
| "completeness": round(completeness, 3), | |
| "tone": tone, | |
| "already_done": steps_completed, | |
| } | |
| return round(score, 3), grader_info, step_identified |