Spaces:
Sleeping
Sleeping
jarvisemitra
SHL Assessment Recommender - Full implementation with hybrid retrieval, slot-based conversation analysis, safety guards, and evaluation harness
ae3c639 | """ | |
| Conversation analyzer: slot extraction and intent classification. | |
| Extracts structured slots from conversation history (stateless — re-extracts | |
| every call) and classifies the user's intent to determine agent behavior. | |
| """ | |
| import re | |
| from app.models import Message | |
| # Slot definitions with extraction patterns | |
| SENIORITY_KEYWORDS = { | |
| "entry": ["entry-level", "entry level", "junior", "fresher", "graduate", "intern", "trainee"], | |
| "mid": ["mid-level", "mid level", "mid-professional", "3-5 years", "4 years", "intermediate"], | |
| "senior": ["senior", "lead", "principal", "staff", "5+ years", "7+ years", "10+ years", "experienced"], | |
| "executive": ["director", "executive", "cxo", "c-suite", "vp", "vice president", "head of", "15 years"], | |
| "manager": ["manager", "supervisor", "team lead", "front line manager"], | |
| } | |
| ASSESSMENT_PURPOSE = { | |
| "selection": ["hiring", "screening", "selection", "recruit", "candidate", "fill", "hire", "assess candidates"], | |
| "development": ["development", "re-skill", "reskill", "upskill", "talent audit", "training", "coaching", "growth"], | |
| } | |
| class ConversationSlots: | |
| """Extracted slots from conversation history.""" | |
| def __init__(self): | |
| self.role: str | None = None # e.g., "Java developer", "contact center agent" | |
| self.seniority: str | None = None # entry, mid, senior, executive, manager | |
| self.technical_skills: list[str] = [] # e.g., ["java", "spring", "sql"] | |
| self.behavioral_skills: list[str] = [] # e.g., ["leadership", "communication"] | |
| self.industry: str | None = None # e.g., "healthcare", "manufacturing" | |
| self.language: str | None = None # e.g., "English", "Spanish" | |
| self.volume: str | None = None # e.g., "500", "high volume" | |
| self.purpose: str | None = None # selection or development | |
| self.constraints: list[str] = [] # e.g., ["quick", "short", "no personality"] | |
| self.specific_requests: list[str] = [] # e.g., ["add personality", "remove REST"] | |
| def confidence(self) -> float: | |
| """ | |
| Confidence score (0-1) based on how many slots are filled. | |
| Used to decide whether to clarify or recommend. | |
| """ | |
| filled = 0 | |
| total = 5 # role, seniority, skills (tech or behavioral), purpose are most important | |
| if self.role: | |
| filled += 1 | |
| if self.seniority: | |
| filled += 1 | |
| if self.technical_skills or self.behavioral_skills: | |
| filled += 1 | |
| if self.purpose: | |
| filled += 0.5 | |
| if self.industry: | |
| filled += 0.5 | |
| return min(filled / total, 1.0) | |
| def to_summary(self) -> str: | |
| """Summarize slots for LLM context.""" | |
| parts = [] | |
| if self.role: | |
| parts.append(f"Role: {self.role}") | |
| if self.seniority: | |
| parts.append(f"Seniority: {self.seniority}") | |
| if self.technical_skills: | |
| parts.append(f"Technical Skills: {', '.join(self.technical_skills)}") | |
| if self.behavioral_skills: | |
| parts.append(f"Behavioral Skills: {', '.join(self.behavioral_skills)}") | |
| if self.industry: | |
| parts.append(f"Industry: {self.industry}") | |
| if self.language: | |
| parts.append(f"Language: {self.language}") | |
| if self.volume: | |
| parts.append(f"Volume: {self.volume}") | |
| if self.purpose: | |
| parts.append(f"Purpose: {self.purpose}") | |
| if self.constraints: | |
| parts.append(f"Constraints: {', '.join(self.constraints)}") | |
| if self.specific_requests: | |
| parts.append(f"Specific Requests: {', '.join(self.specific_requests)}") | |
| return "\n".join(parts) if parts else "No slots extracted yet." | |
| def build_search_queries(self) -> list[str]: | |
| """Build search queries from filled slots for retrieval.""" | |
| queries = [] | |
| # Primary query: role + skills | |
| primary_parts = [] | |
| if self.role: | |
| primary_parts.append(self.role) | |
| if self.technical_skills: | |
| primary_parts.append(" ".join(self.technical_skills)) | |
| if primary_parts: | |
| queries.append(" ".join(primary_parts)) | |
| # Technical skills query | |
| if self.technical_skills: | |
| queries.append(" ".join(self.technical_skills)) | |
| # Behavioral query | |
| if self.behavioral_skills: | |
| queries.append(" ".join(self.behavioral_skills)) | |
| # Seniority + role query | |
| if self.seniority and self.role: | |
| queries.append(f"{self.seniority} {self.role}") | |
| # Industry-specific query | |
| if self.industry: | |
| queries.append(self.industry) | |
| # Specific requests | |
| for req in self.specific_requests: | |
| queries.append(req) | |
| # Fallback: use the latest user message as-is | |
| if not queries: | |
| queries.append("assessment") | |
| return queries | |
| class ConversationAnalyzer: | |
| """ | |
| Analyzes conversation history to extract slots and classify intent. | |
| Stateless: re-analyzes from scratch every call. | |
| """ | |
| def extract_slots(self, messages: list[Message]) -> ConversationSlots: | |
| """Extract all slots from the full conversation history.""" | |
| slots = ConversationSlots() | |
| # Combine all user messages for analysis | |
| user_texts = [m.content for m in messages if m.role == "user"] | |
| combined = " ".join(user_texts).lower() | |
| # Extract seniority | |
| for level, keywords in SENIORITY_KEYWORDS.items(): | |
| for kw in keywords: | |
| if kw in combined: | |
| slots.seniority = level | |
| break | |
| if slots.seniority: | |
| break | |
| # Extract purpose | |
| for purpose, keywords in ASSESSMENT_PURPOSE.items(): | |
| for kw in keywords: | |
| if kw in combined: | |
| slots.purpose = purpose | |
| break | |
| # Extract language requirements | |
| language_patterns = [ | |
| (r"spanish|español", "Spanish"), | |
| (r"french|français", "French"), | |
| (r"german|deutsch", "German"), | |
| (r"portuguese|português", "Portuguese"), | |
| (r"chinese|mandarin", "Chinese"), | |
| (r"english", "English"), | |
| ] | |
| for pattern, lang in language_patterns: | |
| if re.search(pattern, combined): | |
| slots.language = lang | |
| # Extract volume | |
| volume_match = re.search(r'(\d+)\s+(candidates|agents|people|staff|hires)', combined) | |
| if volume_match: | |
| slots.volume = volume_match.group(0) | |
| elif "high.volume" in combined or "volume" in combined: | |
| slots.volume = "high volume" | |
| # The role, technical skills, and specific requests are best extracted | |
| # by the LLM as part of the agent call — rule-based extraction is too brittle | |
| # for diverse job descriptions. We provide the raw text to the LLM prompt. | |
| return slots | |
| def classify_intent(self, messages: list[Message]) -> str: | |
| """ | |
| Classify the user's intent from the latest message + context. | |
| Returns one of: | |
| 'initial' — first message, may need clarification | |
| 'providing_info' — user is answering a clarification question | |
| 'refine' — user wants to add/remove/change recommendations | |
| 'compare' — user is asking about differences | |
| 'confirm' — user is accepting the current list | |
| 'off_topic' — not about SHL assessments | |
| """ | |
| if not messages: | |
| return "initial" | |
| latest = messages[-1].content.lower() | |
| # Check for comparison intent | |
| compare_patterns = [ | |
| r"what('?s|\s+is)\s+the\s+difference", | |
| r"compare\b", | |
| r"difference\s+between", | |
| r"how\s+(does|do|is)\s+.+\s+(differ|compare|versus|vs)", | |
| r"\bvs\.?\b", | |
| ] | |
| for pattern in compare_patterns: | |
| if re.search(pattern, latest): | |
| return "compare" | |
| # Check for refinement intent | |
| refine_patterns = [ | |
| r"\badd\b", | |
| r"\bremove\b", | |
| r"\bdrop\b", | |
| r"\breplace\b", | |
| r"\bswap\b", | |
| r"\bupdate\b", | |
| r"\bchange\b", | |
| r"\bactually\b", | |
| r"\binstead\b", | |
| r"\balso\b.*\b(include|test|assess)", | |
| ] | |
| for pattern in refine_patterns: | |
| if re.search(pattern, latest): | |
| return "refine" | |
| # Check for confirmation | |
| confirm_patterns = [ | |
| r"^(perfect|great|good|thanks|thank|ok|okay|confirmed?|locking|lock\s+it|that('?s| is)\s+(it|what|good|perfect|fine))", | |
| r"keep\s+(it|the|this)", | |
| r"that\s+covers\s+it", | |
| r"we('?ll|\s+will)\s+use", | |
| r"final\s+list", | |
| ] | |
| for pattern in confirm_patterns: | |
| if re.search(pattern, latest): | |
| return "confirm" | |
| # Check if it's the first message | |
| user_messages = [m for m in messages if m.role == "user"] | |
| if len(user_messages) <= 1: | |
| return "initial" | |
| return "providing_info" | |
| def has_recommendations_in_history(self, messages: list[Message]) -> bool: | |
| """Check if the agent has already provided recommendations.""" | |
| for msg in messages: | |
| if msg.role == "assistant": | |
| # Look for recommendation patterns in assistant messages | |
| if any( | |
| indicator in msg.content.lower() | |
| for indicator in ["here are", "shortlist", "battery", "recommend", "assessment"] | |
| ): | |
| return True | |
| return False | |
| def get_latest_user_message(self, messages: list[Message]) -> str: | |
| """Get the most recent user message.""" | |
| for msg in reversed(messages): | |
| if msg.role == "user": | |
| return msg.content | |
| return "" | |