""" agents/scorer.py ──────────────────────────────────────────────────────────────────────────── ScorerAgent: Semantic, context-aware answer scoring — implements Agenda Item #4. Optimized to remove visibility of the coverage percentage metric entirely. """ import json class ScorerAgent: """ Scores a candidate's interview answer using semantic STAR methodology evaluation. """ _MIN_ANSWER_LEN = 15 def __init__(self, llm_fn): """ Args: llm_fn: Callable (prompt: str, temperature: float, max_tokens: int) → str """ self._ask = llm_fn # ── Public entry point ──────────────────────────────────────────────────── def run(self, answer: str, question: str, job_profile: dict) -> dict: """ Score the candidate's answer using deep semantic and contextual analysis. """ if not answer or len(answer.strip()) < self._MIN_ANSWER_LEN: return self._short_answer_result() keywords = job_profile.get("keywords", []) industry = job_profile.get("industry", "General") role_level = job_profile.get("role_level", "Mid-Level") answer_clip = answer.strip()[:600] # ── Step 1: Semantic Prompt Execution ───────────────────────────────── prompt = self._build_prompt(answer_clip, question, industry, role_level, keywords) # Call your core engine (now running Qwen 2.5 7B via Chat Completion) llm_response = self._ask(prompt, temperature=0.1, max_tokens=400) # ── Step 2: Parse and Extract Structured JSON Metrics ───────────────── parsed_data = self._parse_json_response(llm_response, keywords) # ── Step 3: Fast STAR Heuristic ─────────────────────────────────────── answer_lower = answer_clip.lower() star_words = ["situation", "task", "action", "result", "outcome", "challenge", "i did", "i then", "as a result"] star_hint = sum(1 for w in star_words if w in answer_lower) < 2 # Map semantic updates directly to your existing UI display schema return { "raw_feedback": parsed_data["feedback_str"], "score_str": f"{parsed_data['score']}/10", "numeric_score": float(parsed_data["score"]), "hit_keywords": parsed_data["hit_keywords"], "missed_keywords": parsed_data["missed_keywords"], "coverage_pct": 0.0, # 🚨 FORCE ZERO TO STRIP PERCENT METRICS FROM THE GRADIO PAYLOAD "star_hint": star_hint, } # ── Advanced Prompt Builder ─────────────────────────────────────────────── def _build_prompt(self, answer: str, question: str, industry: str, role_level: str, keywords: list) -> str: keywords_str = ", ".join(keywords) if keywords else "General Industry Domain" return f"""You are an elite technical interviewer. Evaluate the candidate's response using the STAR methodology. Interview Question: {question} Candidate's Answer: {answer} Target Domain Keywords: {keywords_str} 🚨 CRITICAL KEYWORD EVALUATION RULES (READ CAREFULLY): 1. Do NOT look for exact text characters or verbatim matches. 2. If the candidate describes the concept of a keyword, mentions a synonym, or explains the real-world task/workflow related to that keyword, you MUST count it as a SUCCESSFUL match. 3. For example: If the keyword is "Database", and they discuss "storing text files" or "SQL optimization", that is a successful match. Be highly generous with conceptual matches! Provide your evaluation in a raw, clean JSON format. Do not include any markdown block formatting (like ```json). Use exactly this structure: {{ "score": [Integer from 1 to 10 based on content quality], "hit_keywords": [Array of string keywords from the Target list that were covered or conceptually explained], "missed_keywords": [Array of string keywords from the Target list completely missing in both text and concept], "strength": "One clear sentence highlighting what the candidate did well.", "weakness": "One clear sentence identifying the biggest structural or domain gap.", "fix": "One clear sentence containing actionable advice to improve the response." }}""" # ── Robust Safe JSON Parser ─────────────────────────────────────────────── def _parse_json_response(self, raw_text: str, original_keywords: list) -> dict: """Safely parses LLM output and provides bulletproof defaults if parsing fails.""" try: # Clean off any accidental markdown enclosing blocks clean_text = raw_text.replace("```json", "").replace("```", "").strip() data = json.loads(clean_text) score = int(data.get("score", 6)) hit_kw = data.get("hit_keywords", []) missed_kw = data.get("missed_keywords", []) # Reconstruct the UI display text string dynamically without keyword metric lines feedback_str = ( f"Relevant: YES\n" f"Score: {score}/10\n" f"Strength: {data.get('strength', 'Good conceptual delivery.')}\n" f"Weakness: {data.get('weakness', 'Some technical elements can be expanded.')}\n" f"Fix: {data.get('fix', 'Incorporate more structural milestones in your history.')}" ) return { "score": score, "hit_keywords": hit_kw, "missed_keywords": missed_kw, "coverage_pct": 0.0, # 🚨 BYPASS THE MATH CALCULATION COMPLETELY "feedback_str": feedback_str } except Exception: # Safe production fallback configuration return { "score": 6, "hit_keywords": original_keywords[:1] if original_keywords else [], "missed_keywords": original_keywords[1:] if original_keywords else [], "coverage_pct": 0.0, # 🚨 BYPASS THE MATH CALCULATION COMPLETELY "feedback_str": "Relevant: YES\nScore: 6/10\nStrength: Answer was recorded successfully.\nWeakness: Evaluation parsing exception occurred.\nFix: Try expanding your STAR answer metrics slightly." } @staticmethod def _short_answer_result() -> dict: return { "raw_feedback": "Please write a more detailed answer (at least 15 characters).", "score_str": "", "numeric_score": None, "hit_keywords": [], "missed_keywords": [], "coverage_pct": 0.0, "star_hint": True, }