SLSAGENT / app /analyzer.py
jarvisemitra
SHL Assessment Recommender - Full implementation with hybrid retrieval, slot-based conversation analysis, safety guards, and evaluation harness
ae3c639
Raw
History Blame Contribute Delete
10.3 kB
"""
Conversation analyzer: slot extraction and intent classification.
Extracts structured slots from conversation history (stateless — re-extracts
every call) and classifies the user's intent to determine agent behavior.
"""
import re
from app.models import Message
# Slot definitions with extraction patterns
SENIORITY_KEYWORDS = {
"entry": ["entry-level", "entry level", "junior", "fresher", "graduate", "intern", "trainee"],
"mid": ["mid-level", "mid level", "mid-professional", "3-5 years", "4 years", "intermediate"],
"senior": ["senior", "lead", "principal", "staff", "5+ years", "7+ years", "10+ years", "experienced"],
"executive": ["director", "executive", "cxo", "c-suite", "vp", "vice president", "head of", "15 years"],
"manager": ["manager", "supervisor", "team lead", "front line manager"],
}
ASSESSMENT_PURPOSE = {
"selection": ["hiring", "screening", "selection", "recruit", "candidate", "fill", "hire", "assess candidates"],
"development": ["development", "re-skill", "reskill", "upskill", "talent audit", "training", "coaching", "growth"],
}
class ConversationSlots:
"""Extracted slots from conversation history."""
def __init__(self):
self.role: str | None = None # e.g., "Java developer", "contact center agent"
self.seniority: str | None = None # entry, mid, senior, executive, manager
self.technical_skills: list[str] = [] # e.g., ["java", "spring", "sql"]
self.behavioral_skills: list[str] = [] # e.g., ["leadership", "communication"]
self.industry: str | None = None # e.g., "healthcare", "manufacturing"
self.language: str | None = None # e.g., "English", "Spanish"
self.volume: str | None = None # e.g., "500", "high volume"
self.purpose: str | None = None # selection or development
self.constraints: list[str] = [] # e.g., ["quick", "short", "no personality"]
self.specific_requests: list[str] = [] # e.g., ["add personality", "remove REST"]
@property
def confidence(self) -> float:
"""
Confidence score (0-1) based on how many slots are filled.
Used to decide whether to clarify or recommend.
"""
filled = 0
total = 5 # role, seniority, skills (tech or behavioral), purpose are most important
if self.role:
filled += 1
if self.seniority:
filled += 1
if self.technical_skills or self.behavioral_skills:
filled += 1
if self.purpose:
filled += 0.5
if self.industry:
filled += 0.5
return min(filled / total, 1.0)
def to_summary(self) -> str:
"""Summarize slots for LLM context."""
parts = []
if self.role:
parts.append(f"Role: {self.role}")
if self.seniority:
parts.append(f"Seniority: {self.seniority}")
if self.technical_skills:
parts.append(f"Technical Skills: {', '.join(self.technical_skills)}")
if self.behavioral_skills:
parts.append(f"Behavioral Skills: {', '.join(self.behavioral_skills)}")
if self.industry:
parts.append(f"Industry: {self.industry}")
if self.language:
parts.append(f"Language: {self.language}")
if self.volume:
parts.append(f"Volume: {self.volume}")
if self.purpose:
parts.append(f"Purpose: {self.purpose}")
if self.constraints:
parts.append(f"Constraints: {', '.join(self.constraints)}")
if self.specific_requests:
parts.append(f"Specific Requests: {', '.join(self.specific_requests)}")
return "\n".join(parts) if parts else "No slots extracted yet."
def build_search_queries(self) -> list[str]:
"""Build search queries from filled slots for retrieval."""
queries = []
# Primary query: role + skills
primary_parts = []
if self.role:
primary_parts.append(self.role)
if self.technical_skills:
primary_parts.append(" ".join(self.technical_skills))
if primary_parts:
queries.append(" ".join(primary_parts))
# Technical skills query
if self.technical_skills:
queries.append(" ".join(self.technical_skills))
# Behavioral query
if self.behavioral_skills:
queries.append(" ".join(self.behavioral_skills))
# Seniority + role query
if self.seniority and self.role:
queries.append(f"{self.seniority} {self.role}")
# Industry-specific query
if self.industry:
queries.append(self.industry)
# Specific requests
for req in self.specific_requests:
queries.append(req)
# Fallback: use the latest user message as-is
if not queries:
queries.append("assessment")
return queries
class ConversationAnalyzer:
"""
Analyzes conversation history to extract slots and classify intent.
Stateless: re-analyzes from scratch every call.
"""
def extract_slots(self, messages: list[Message]) -> ConversationSlots:
"""Extract all slots from the full conversation history."""
slots = ConversationSlots()
# Combine all user messages for analysis
user_texts = [m.content for m in messages if m.role == "user"]
combined = " ".join(user_texts).lower()
# Extract seniority
for level, keywords in SENIORITY_KEYWORDS.items():
for kw in keywords:
if kw in combined:
slots.seniority = level
break
if slots.seniority:
break
# Extract purpose
for purpose, keywords in ASSESSMENT_PURPOSE.items():
for kw in keywords:
if kw in combined:
slots.purpose = purpose
break
# Extract language requirements
language_patterns = [
(r"spanish|español", "Spanish"),
(r"french|français", "French"),
(r"german|deutsch", "German"),
(r"portuguese|português", "Portuguese"),
(r"chinese|mandarin", "Chinese"),
(r"english", "English"),
]
for pattern, lang in language_patterns:
if re.search(pattern, combined):
slots.language = lang
# Extract volume
volume_match = re.search(r'(\d+)\s+(candidates|agents|people|staff|hires)', combined)
if volume_match:
slots.volume = volume_match.group(0)
elif "high.volume" in combined or "volume" in combined:
slots.volume = "high volume"
# The role, technical skills, and specific requests are best extracted
# by the LLM as part of the agent call — rule-based extraction is too brittle
# for diverse job descriptions. We provide the raw text to the LLM prompt.
return slots
def classify_intent(self, messages: list[Message]) -> str:
"""
Classify the user's intent from the latest message + context.
Returns one of:
'initial' — first message, may need clarification
'providing_info' — user is answering a clarification question
'refine' — user wants to add/remove/change recommendations
'compare' — user is asking about differences
'confirm' — user is accepting the current list
'off_topic' — not about SHL assessments
"""
if not messages:
return "initial"
latest = messages[-1].content.lower()
# Check for comparison intent
compare_patterns = [
r"what('?s|\s+is)\s+the\s+difference",
r"compare\b",
r"difference\s+between",
r"how\s+(does|do|is)\s+.+\s+(differ|compare|versus|vs)",
r"\bvs\.?\b",
]
for pattern in compare_patterns:
if re.search(pattern, latest):
return "compare"
# Check for refinement intent
refine_patterns = [
r"\badd\b",
r"\bremove\b",
r"\bdrop\b",
r"\breplace\b",
r"\bswap\b",
r"\bupdate\b",
r"\bchange\b",
r"\bactually\b",
r"\binstead\b",
r"\balso\b.*\b(include|test|assess)",
]
for pattern in refine_patterns:
if re.search(pattern, latest):
return "refine"
# Check for confirmation
confirm_patterns = [
r"^(perfect|great|good|thanks|thank|ok|okay|confirmed?|locking|lock\s+it|that('?s| is)\s+(it|what|good|perfect|fine))",
r"keep\s+(it|the|this)",
r"that\s+covers\s+it",
r"we('?ll|\s+will)\s+use",
r"final\s+list",
]
for pattern in confirm_patterns:
if re.search(pattern, latest):
return "confirm"
# Check if it's the first message
user_messages = [m for m in messages if m.role == "user"]
if len(user_messages) <= 1:
return "initial"
return "providing_info"
def has_recommendations_in_history(self, messages: list[Message]) -> bool:
"""Check if the agent has already provided recommendations."""
for msg in messages:
if msg.role == "assistant":
# Look for recommendation patterns in assistant messages
if any(
indicator in msg.content.lower()
for indicator in ["here are", "shortlist", "battery", "recommend", "assessment"]
):
return True
return False
def get_latest_user_message(self, messages: list[Message]) -> str:
"""Get the most recent user message."""
for msg in reversed(messages):
if msg.role == "user":
return msg.content
return ""