src/loan_assistant.py

"""
Loan Application Assistant - Multi-turn Conversational Agent
This module handles the conversational flow for collecting loan application information.
"""

import json
import re
from typing import Dict, List, Tuple, Optional, Any
from dataclasses import dataclass, field
from enum import Enum
from ab_config import config
import pandas as pd
import numpy as np
import difflib
from xai_methods import get_friendly_feature_name, FEATURE_DISPLAY_NAMES

# Import natural conversation enhancer
try:
    from natural_conversation import enhance_response, enhance_validation_message, handle_meta_question
    NATURAL_CONVERSATION_AVAILABLE = True
except ImportError:
    NATURAL_CONVERSATION_AVAILABLE = False
    def enhance_response(response, context=None, response_type="loan"):
        return response
    def enhance_validation_message(field, user_input, expected_format, attempt=1):
        return None  # Fallback returns None to use hardcoded messages
    def handle_meta_question(field, user_input, high_anthropomorphism=True):
        return None  # Fallback returns None

class ConversationState(Enum):
    GREETING = "greeting"
    COLLECTING_INFO = "collecting_info"
    REVIEWING = "reviewing"
    PROCESSING = "processing"
    COMPLETE = "complete"
    EXPLAINING = "explaining"

@dataclass
class LoanApplication:
    # Personal Information
    age: Optional[int] = None
    marital_status: Optional[str] = None
    
    # Employment Information
    workclass: Optional[str] = None
    occupation: Optional[str] = None
    hours_per_week: Optional[int] = None
    
    # Education
    education: Optional[str] = None
    education_num: Optional[int] = None
    
    # Financial Information
    capital_gain: Optional[int] = None
    capital_loss: Optional[int] = None
    
    # Demographics
    race: Optional[str] = None
    sex: Optional[str] = None
    native_country: Optional[str] = None
    relationship: Optional[str] = None
    
    # Application metadata
    is_complete: bool = False
    completion_percentage: float = 0.0
    loan_approved: Optional[bool] = None
    
    def to_dict(self) -> Dict[str, Any]:
        """Convert to dictionary for model prediction"""
        return {
            'age': self.age,
            'workclass': self.workclass,
            'education': self.education,
            'education_num': self.education_num,
            'marital_status': self.marital_status,
            'occupation': self.occupation,
            'relationship': self.relationship,
            'race': self.race,
            'sex': self.sex,
            'capital_gain': self.capital_gain or 0,
            'capital_loss': self.capital_loss or 0,
            'hours_per_week': self.hours_per_week,
            'native_country': self.native_country
        }
    
    def calculate_completion(self) -> float:
        """Calculate completion percentage"""
        total_fields = 13
        completed_fields = sum(1 for field_name, field_def in self.__dataclass_fields__.items() 
                              if field_name not in ['is_complete', 'completion_percentage'] 
                              and getattr(self, field_name) is not None)
        self.completion_percentage = (completed_fields / total_fields) * 100
        self.is_complete = self.completion_percentage >= 80  # 80% completion threshold
        return self.completion_percentage

class LoanAssistant:
    def __init__(self, agent):
        self.agent = agent
        self.conversation_state = ConversationState.GREETING
        self.application = LoanApplication()
        self.conversation_history = []
        self.current_field = None
        self.field_attempts = {}
        self.last_shap_result = None  # Store SHAP results for visualization
        self.show_what_if_lab = False  # Show What‑if Lab in UI when user asks what-if
        # Allowed categorical values (canonical forms)
        self.allowed_values = {
            'workclass': ['Private', 'Self-emp-not-inc', 'Self-emp-inc', 'Federal-gov', 'Local-gov', 'State-gov', 'Without-pay', 'Never-worked', '?'],
            'education': ['Preschool','1st-4th','5th-6th','7th-8th','9th','10th','11th','12th','HS-grad','Some-college','Assoc-voc','Assoc-acdm','Bachelors','Masters','Prof-school','Doctorate'],
            'marital_status': ['Married-civ-spouse', 'Divorced', 'Never-married', 'Separated', 'Widowed', 'Married-spouse-absent', 'Married-AF-spouse'],
            'occupation': ['Tech-support','Craft-repair','Other-service','Sales','Exec-managerial','Prof-specialty','Handlers-cleaners','Machine-op-inspct','Adm-clerical','Farming-fishing','Transport-moving','Priv-house-serv','Protective-serv','Armed-Forces','?'],
            'sex': ['Male', 'Female'],
            'race': ['Black', 'Asian-Pac-Islander', 'Amer-Indian-Eskimo', 'White', 'Other'],
            'native_country': ['United-States', 'Cambodia', 'Canada', 'China', 'Columbia', 'Cuba', 'Dominican-Republic', 'Ecuador', 'El-Salvador', 'England', 'France', 'Germany', 'Greece', 'Guatemala', 'Haiti', 'Holand-Netherlands', 'Honduras', 'Hong', 'Hungary', 'India', 'Iran', 'Ireland', 'Italy', 'Jamaica', 'Japan', 'Laos', 'Mexico', 'Nicaragua', 'Outlying-US(Guam-USVI-etc)', 'Peru', 'Philippines', 'Poland', 'Portugal', 'Puerto-Rico', 'Scotland', 'South', 'Taiwan', 'Thailand', 'Trinadad&Tobago', 'Vietnam', 'Yugoslavia', '?'],
            'relationship': ['Wife', 'Own-child', 'Husband', 'Not-in-family', 'Other-relative', 'Unmarried']
        }
        
        # Field collection order and prompts
        self.field_order = [
            'age', 'workclass', 'education', 'marital_status', 
            'occupation', 'hours_per_week', 'sex', 'race',
            'native_country', 'relationship', 'capital_gain', 'capital_loss'
        ]
        
        self.field_prompts = {
            'age': "What's your age?",
            'workclass': "What's your employment type? (e.g., Private sector, Self-employed, Federal/Local/State government, etc.)",
            'education': "What's your highest education level? (e.g., Bachelor's, High school graduate, Master's, Some college, Associate's degree, Doctorate, etc.)",
            'marital_status': "What's your marital status? (e.g., Married, Divorced, Never married, Separated, Widowed, etc.)",
            'occupation': "What's your occupation? (e.g., Tech support, Sales, Professional, Management, Administrative, Farming, etc.)",
            'hours_per_week': "How many hours per week do you work?",
            'sex': "What's your gender? (Male/Female)",
            'race': "What's your race? (e.g., White, Asian-Pacific Islander, Indigenous American, Black, Other)",
            'native_country': "What's your native country? (e.g., United States, Canada, England, Germany, etc.)",
            'relationship': "What's your relationship status? (e.g., Wife, Own-child, Husband, Not in family, Other relative, Unmarried)",
            'capital_gain': "Enter your capital gains for this year. Range: -5000 (losses) to 9000. Enter 0 if none.",
            'capital_loss': "Do you have any capital losses this year? (Enter amount or 0 if none)"
        }
        
        self.validation_rules = {
            'age': {'type': 'int', 'min': 17, 'max': 90},
            'hours_per_week': {'type': 'int', 'min': 1, 'max': 99},
            'capital_gain': {'type': 'int', 'min': -5000, 'max': 9000},
            'capital_loss': {'type': 'int', 'min': 0, 'max': 4356},
            'education_num': {'type': 'int', 'min': 1, 'max': 16}
        }
        
        # Step mapping for progress tracking
        self.field_to_step = {
            'age': 1, 'sex': 2,  # Personal Info
            'workclass': 3, 'occupation': 4, 'hours_per_week': 5,  # Employment
            'education': 6,  # Education
            'capital_gain': 7, 'capital_loss': 8,  # Financial
            'native_country': 9, 'marital_status': 10, 'relationship': 10, 'race': 10  # Background
        }
        
        self.step_descriptions = {
            1: "Personal Info - Age", 2: "Personal Info - Gender",
            3: "Employment - Work Class", 4: "Employment - Occupation", 5: "Employment - Hours",
            6: "Education Level",
            7: "Financial - Capital Gains", 8: "Financial - Capital Losses",
            9: "Background - Country", 10: "Background - Demographics"
        }

    # ----- Helper methods for version-aware messaging -----
    def _is_v1(self) -> bool:
        try:
            return getattr(config, 'version', 'v0') == 'v1'
        except Exception:
            return False

    def _pretty_field(self, field: str) -> str:
        return field.replace('_', ' ').title()

    def _format_error(self, field: str, base_msg: str, allowed: Optional[List[str]] = None) -> str:
        if not self._is_v1():
            # v0: concise, technical
            if allowed:
                return f"{base_msg} Valid options are: {', '.join(allowed)}."
            return base_msg
        # v1: anthropomorphic tone
        msg = f"I might be misreading this. For {self._pretty_field(field)}, {base_msg}"
        if allowed:
            sample = allowed[:8]
            more = " (+ more)" if len(allowed) > 8 else ""
            msg += f"\n\nHere are some examples I can accept: {', '.join(sample)}{more}."
        msg += "\nIf you'd like, I can list all choices or you can pick from the buttons."
        return msg

    def _format_warning(self, field: str, warn_msg: str) -> str:
        if not self._is_v1():
            return warn_msg
        return f"Quick heads‑up about {self._pretty_field(field)}: {warn_msg} If that was intentional, we’ll proceed; otherwise feel free to adjust."

    def _suggest_categorical(self, field: str, value: str, n: int = 3) -> List[str]:
        """Suggest close categorical options using fuzzy match."""
        allowed = self.allowed_values.get(field, [])
        if not allowed:
            return []
        return difflib.get_close_matches(str(value), allowed, n=n, cutoff=0.6)

    def handle_message(self, user_input: str) -> str:
        """Main message handler with enhanced natural conversation and XAI routing"""
        if not user_input.strip():
            return "I didn't catch that. Could you please say something?"
        
        # Check for XAI questions regardless of current state once a decision has been presented
        user_lower = user_input.lower()
        xai_keywords = ['what if', 'why', 'explain', 'how', 'which factors', 'feature importance',
                        'counterfactual', 'simple rules', 'rule-based', 'rule based', 'rules', 'anchor',
                        'what changes', 'what would happen']

        # If we've already provided a decision (loan_approved is True/False), allow immediate XAI routing
        if self.application.loan_approved is not None and any(keyword in user_lower for keyword in xai_keywords):
            return self._handle_explanation(user_input)
        
        # Route to appropriate handler based on conversation state
        if self.conversation_state == ConversationState.GREETING:
            return self._handle_greeting(user_input)
        elif self.conversation_state == ConversationState.COLLECTING_INFO:
            return self._handle_info_collection(user_input)
        elif self.conversation_state == ConversationState.REVIEWING:
            return self._handle_review(user_input)
        elif self.conversation_state == ConversationState.PROCESSING:
            return self._handle_processing(user_input)
        elif self.conversation_state == ConversationState.COMPLETE:
            return self._handle_complete(user_input)
        elif self.conversation_state == ConversationState.EXPLAINING:
            return self._handle_explanation(user_input)
        else:
            return "I'm not sure how to help with that. Please try again."

    def _handle_greeting(self, user_input: str) -> str:
        """Handle initial greeting and start application process"""
        greeting_keywords = ['hi', 'hello', 'hey', 'start', 'apply', 'loan', 'application']
        
        if config.show_anthropomorphic:
            # High anthropomorphism: Warm, personal, conversational
            if any(keyword in user_input.lower() for keyword in greeting_keywords) or user_input.lower() in ['yes', 'y']:
                self.conversation_state = ConversationState.COLLECTING_INFO
                base_greeting = ("Hello! I'm Luna, your personal loan application assistant. 😊 I will process your information and provide you with your loan qualification results. If you have any questions about the results, feel free to ask!\n\n"
                       "**I will collect information step by step** (not all at once):\n"
                       "• **Step 1-2:** Personal Information (Age, Gender, etc.)\n"
                       "• **Step 3-5:** Employment Details (Work Class, Occupation, Hours)\n"
                       "• **Step 6:** Education Level\n"
                       "• **Step 7-8:** Financial Information (Capital Gains/Losses)\n"
                       "• **Step 9-10:** Background & Relationship Status\n\n"
                       "Let's start with **Step 1:**")
                
                # Enhance greeting with LLM
                if NATURAL_CONVERSATION_AVAILABLE:
                    try:
                        enhanced = enhance_response(base_greeting, {}, "greeting", high_anthropomorphism=True)
                        if enhanced and len(enhanced.strip()) > 20:
                            return f"{enhanced}\n\n{self._get_next_question()}"
                    except Exception:
                        pass
                
                return f"{base_greeting}\n\n{self._get_next_question()}"
            else:
                base_prompt = ("Hi there! I'm Luna, your personal loan application assistant. 😊 I will process your information and provide you with your loan qualification results. If you have any questions about the results, feel free to ask!\n\n"
                       "**I will collect your information step by step** (not all at once)\n"
                       "• **Step 1-2:** Personal Information (Age, Gender, etc.)\n"
                       "• **Step 3-5:** Employment Details (Work Class, Occupation, Hours)\n"
                       "• **Step 6:** Education Level\n"
                       "• **Step 7-8:** Financial Information (Capital Gains/Losses)\n"
                       "• **Step 9-10:** Background & Relationship Status\n\n"
                    
                       "Would you like to start your loan application? Just say 'yes' or 'start' to begin!")
                
                # Enhance with LLM
                if NATURAL_CONVERSATION_AVAILABLE:
                    try:
                        enhanced = enhance_response(base_prompt, {}, "greeting_prompt", high_anthropomorphism=True)
                        if enhanced and len(enhanced.strip()) > 20:
                            return enhanced
                    except Exception:
                        pass
                
                return base_prompt
        else:
            # Low anthropomorphism: Technical, concise, machine-like
            if any(keyword in user_input.lower() for keyword in greeting_keywords) or user_input.lower() in ['yes', 'y']:
                self.conversation_state = ConversationState.COLLECTING_INFO
                base_greeting = ("AI Loan Assistant initialized. I will collect your information and process your loan qualification.\n\n"
                       "**Information collection process** (sequential):\n"
                       "• **Step 1-2:** Personal data (Age, Gender)\n"
                       "• **Step 3-5:** Employment data (Work Class, Occupation, Hours)\n"
                       "• **Step 6:** Education level\n"
                       "• **Step 7-8:** Financial data (Capital Gains/Losses)\n"
                       "• **Step 9-10:** Demographics & Relationship\n\n"
                       "Progress tracking available in sidebar.\n\n"
                       "**Step 1:**")
                
                # Enhance with LLM for professional tone
                if NATURAL_CONVERSATION_AVAILABLE:
                    try:
                        enhanced = enhance_response(base_greeting, {}, "greeting", high_anthropomorphism=False)
                        if enhanced and len(enhanced.strip()) > 20:
                            return f"{enhanced}\n\n{self._get_next_question()}"
                    except Exception:
                        pass
                
                return f"{base_greeting}\n\n{self._get_next_question()}"
            else:
                base_prompt = ("AI Loan Assistant system ready. I will collect your data and evaluate your loan qualification.\n\n"
                       "**Data collection process** (sequential):\n"
                       "• **Step 1-2:** Personal data (Age, Gender)\n"
                       "• **Step 3-5:** Employment data (Work Class, Occupation, Hours)\n"
                       "• **Step 6:** Education level\n"
                       "• **Step 7-8:** Financial data (Capital Gains/Losses)\n"
                       "• **Step 9-10:** Demographics & Relationship\n\n"
                       "Progress tracking available in sidebar.\n\n"
                       "Enter 'yes' or 'start' to begin data collection.")
                
                # Enhance with LLM for professional tone
                if NATURAL_CONVERSATION_AVAILABLE:
                    try:
                        enhanced = enhance_response(base_prompt, {}, "greeting_prompt", high_anthropomorphism=False)
                        if enhanced and len(enhanced.strip()) > 20:
                            return enhanced
                    except Exception:
                        pass
                
                return base_prompt

    def _handle_info_collection(self, user_input: str) -> str:
        """Handle information collection phase"""
        if user_input.lower() in ['quit', 'exit', 'stop', 'cancel']:
            if config.show_anthropomorphic:
                return "Application cancelled. Feel free to start again anytime by saying 'hi'! 😊"
            else:
                return "Application process terminated. Restart available via 'hi' command."
        
        if user_input.lower() in ['review', 'check', 'status']:
            return self._show_progress()
        
        # Handle '?' - provide copyable list of valid options
        if user_input.strip() == '?' and self.current_field:
            return self._get_copyable_options(self.current_field)
        
        if user_input.lower() in ['help', 'help me', 'what do i do', 'stuck', 'confused']:
            if self.current_field:
                if config.show_anthropomorphic:
                    return f"No problem! Let me help you with {self.current_field.replace('_', ' ')}:\n\n{self._get_field_help(self.current_field)}\n\n✨ You can also use the quick-select buttons if available!"
                else:
                    return f"Help for {self.current_field.replace('_', ' ')}:\n\n{self._get_field_help(self.current_field)}\n\nQuick-select buttons available when applicable."
            else:
                if config.show_anthropomorphic:
                    return "I'm here to help! I'm collecting information for your loan application step by step. You can say 'review' to see your progress, or just answer the current question. 😊"
                else:
                    return "Data collection in progress. Enter 'review' for status or respond to current query."
        
        # Process the current field
        if self.current_field:
            # Store the field we're processing BEFORE it gets changed
            completed_field = self.current_field
            result = self._process_field_input(self.current_field, user_input)
            if result['success']:
                completion = self.application.calculate_completion()
                
                # Check if we have all required information
                if self.application.is_complete:
                    self.conversation_state = ConversationState.REVIEWING
                    if config.show_anthropomorphic:
                        return (f"🎉 Fantastic! I've collected all the necessary information ({completion:.0f}% complete).\n\n"
                               + self._show_application_summary() + 
                               "\n\n💼 Would you like me to process your loan application now? (yes/no)")
                    else:
                        return (f"Data collection complete ({completion:.0f}%).\n\n"
                               + self._show_application_summary() + 
                               "\n\nProcess application? (yes/no)")
                else:
                    next_question = self._get_next_question()
                    if next_question:
                        # Get step information for the NEXT field (self.current_field is now updated)
                        next_step = self.field_to_step.get(self.current_field, 0) if self.current_field else 0
                        # Get the value that was just set for the COMPLETED field
                        completed_value = str(getattr(self.application, completed_field, ''))
                        # Use LLM-enhanced success message with the correct completed field
                        success_msg = self._get_success_message(
                            completed_field,
                            completed_value,
                            completion,
                            next_step
                        )
                        return f"{success_msg}\n{next_question}"
                    else:
                        # Fallback if no next question
                        self.conversation_state = ConversationState.REVIEWING
                        return (f"Great! I have most of the information ({completion:.0f}% complete).\n\n"
                               + self._show_application_summary() + 
                               "\n\nWould you like me to process your loan application? (yes/no)")
            else:
                return result['message']
        else:
            return self._get_next_question()

    def _handle_review(self, user_input: str) -> str:
        """Handle application review phase"""
        if user_input.lower() in ['yes', 'y', 'proceed', 'process', 'submit']:
            self.conversation_state = ConversationState.PROCESSING
            return self._process_application()
        elif user_input.lower() in ['no', 'n', 'edit', 'change', 'modify']:
            self.conversation_state = ConversationState.COLLECTING_INFO
            return ("No problem! What would you like to change? You can say things like:\n"
                   "- 'Change my age to 30'\n"
                   "- 'Update my occupation'\n"
                   "- 'My education is Masters'\n\n"
                   "Or just tell me which field you'd like to update.")
        else:
            return ("Please let me know if you'd like to proceed with the application (yes) or make changes (no).\n\n"
                   + self._show_application_summary())

    def _handle_processing(self, user_input: str) -> str:
        """Handle post-processing phase"""
        if self._is_xai_query(user_input):
            self.conversation_state = ConversationState.EXPLAINING
            return self._handle_explanation(user_input)
        elif user_input.lower() in ['new', 'another', 'restart', 'again']:
            return self._restart_application()
        else:
            if config.explanation == "none":
                return ("Your application has been processed! You can:\n"
                       "- Say 'new' to start a new application\n"
                       "- For further details, please visit your local branch")
            else:
                return ("Your application has been processed! Here are your options:\n"
                       "- Say 'explain' to understand how the decision was made\n"
                       "- Say 'new' to start a new application\n"
                       "- Ask me any questions about the result")

    def _handle_complete(self, user_input: str) -> str:
        """Handle interactions when application is complete"""
        user_lower = user_input.lower()
        
        if user_lower in ['new', 'another', 'restart', 'again', 'start over']:
            return self._restart_application()
        elif self._is_xai_query(user_input):
            return self._handle_explanation(user_input)
        else:
            # Check if the application has been processed
            if self.application.loan_approved is None:
                return ("It looks like your application hasn't been processed yet. Please complete the application first, "
                       "or if you believe this is an error, try saying 'start over' to begin a new application.")
            
            # Provide helpful guidance
            result = self.application.loan_approved
            status = "approved" if result else "denied"
            
            if config.explanation == "none":
                # Simple message without explanation options
                return (f"Your loan application has been {status}. "
                       f"For further details, please visit your local branch. "
                       f"Would you like to start a new application? Just say 'new' or 'restart'.")
            else:
                # Full options when explanations are available
                return (f"Your loan application has been {status}! Here's what you can do:\n\n"
                       "🔍 **Ask for explanations:** 'Why was I approved/denied?' or 'Explain the decision'\n"
                       "🔧 **What-if analysis:** 'What if my income was higher?' or 'What changes would help?'\n"
                       "📊 **Feature importance:** 'Which factors were most important?'\n"
                       "🆕 **New application:** 'Start a new application'\n\n"
                       "Just ask me anything about your loan decision!")

    def _handle_explanation(self, user_input: str) -> str:
        """Handle explanation requests using the XAgent-compatible approach"""
        # If explanations are disabled (none condition), provide generic response
        if config.explanation == "none":
            return ("I'm sorry, but detailed explanations are not available at this time. "
                   "For further information about your loan decision, please visit your local branch. "
                   "Would you like to start a new application?")
        
        try:
            # Set up the agent instance properly first
            self._setup_agent_instance()
            
            # Use XAgent-style matching approach
            features = self.agent.data.get('features', [])
            prediction = self.agent.predicted_class
            current_instance = self.agent.current_instance
            labels = self.agent.data.get('classes', [])
            
            # Use the match function like XAgent does
            matched_question = self.agent.nlu_model.match(user_input, features, prediction, current_instance, labels)
            
            # Debug logging
            print(f"🔍 DEBUG: Matched question: {matched_question}")
            print(f"🔍 DEBUG: Input: {user_input}")
            
            if matched_question != "unknown":
                # Good match found - get the label and map to XAI method
                try:
                    import pandas as pd
                    # Get the label for the matched question from the NLU dataframe
                    df_matches = self.agent.nlu_model.df.query('Question == @matched_question')
                    if len(df_matches) > 0:
                        label = df_matches['Label'].iloc[0]
                        xai_method = self.agent.nlu_model.map_label_to_xai_method(label)
                    else:
                        # Fallback: infer method from the matched question text
                        mq = (matched_question or '').lower()
                        if any(k in mq for k in ['rule', 'rule-based', 'rule based', 'anchor', 'simple requirement', 'minimum requirement']):
                            xai_method = 'anchor'
                            label = None
                        elif any(k in mq for k in ['what if', 'change', 'counterfactual']):
                            xai_method = 'dice'
                            label = None
                        else:
                            xai_method = 'shap'
                            label = None
                    
                    print(f"🔍 DEBUG: Label: {label}, XAI method: {xai_method}")
                    
                    # Heuristic override: if mapping returned 'general', infer from input
                    inferred_method = xai_method
                    if xai_method == 'general':
                        ui = (user_input or '').lower()
                        if any(k in ui for k in ['rule', 'rule-based', 'rule based', 'anchor']):
                            inferred_method = 'anchor'
                        elif any(k in ui for k in ['what if', 'change', 'different', 'counterfactual']):
                            inferred_method = 'dice'
                        else:
                            inferred_method = 'shap'
                    
                    # CRITICAL: Remap to available explanation method for this experimental condition
                    # Force all explanation requests to use the method enabled for this condition
                    inferred_method = inferred_method.lower().strip()  # Normalize
                    
                    if config.explanation == 'counterfactual':
                        # Conditions 3-4: Only counterfactual (DiCE) available
                        # Map ANYTHING except anchor to DiCE
                        if inferred_method != 'anchor':
                            print(f"🔍 DEBUG MAIN: Remapping '{inferred_method}' → 'dice' for counterfactual condition")
                            inferred_method = 'dice'
                    elif config.explanation == 'feature_importance':
                        # Conditions 5-6: Only feature importance (SHAP) available
                        # Map ANYTHING except anchor to SHAP
                        if inferred_method != 'anchor':
                            print(f"🔍 DEBUG MAIN: Remapping '{inferred_method}' → 'shap' for feature_importance condition")
                            inferred_method = 'shap'
                    # Note: 'anchor' is always available as a baseline in all conditions

                    # If user is asking what-if (counterfactual), enable What‑if Lab in UI
                    if inferred_method == 'dice':
                        self.show_what_if_lab = True
                    # Create intent result in the format expected by route_to_xai_method
                    intent_result = {
                        'intent': inferred_method,
                        'label': label,
                        'matched_question': matched_question
                    }
                    
                    # Route to the correct XAI method
                    from xai_methods import route_to_xai_method
                    explanation_result = route_to_xai_method(self.agent, intent_result)
                    
                    explanation = explanation_result.get('explanation', 'Sorry, I could not generate an explanation.')
                    
                    # Store SHAP results for visualization (if available)
                    if (inferred_method == 'shap' and 
                        isinstance(explanation_result, dict) and 
                        ('feature_impacts' in explanation_result or 'shap_values' in explanation_result)):
                        self.last_shap_result = explanation_result
                    else:
                        self.last_shap_result = None
                    
                except Exception as e:
                    explanation = f"Sorry, I couldn't generate that explanation right now. Error: {str(e)}"
                
            else:
                # No good match found - fallback to general agent
                # But first apply remapping for experimental conditions
                try:
                    print(f"🔍 DEBUG FALLBACK PATH: user_input='{user_input}'")
                    print(f"🔍 DEBUG CONFIG: explanation={config.explanation}, show_shap={config.show_shap_visualizations}, show_counterfactual={config.show_counterfactual}")
                    
                    intent_result_fallback, confidence, suggestions = self.agent.nlu_model.classify_intent(user_input)
                    
                    print(f"🔍 DEBUG FALLBACK: classify_intent returned: {intent_result_fallback}")
                    
                    if isinstance(intent_result_fallback, dict) and 'intent' in intent_result_fallback:
                        inferred_method = intent_result_fallback['intent'].lower().strip()
                        
                        print(f"🔍 DEBUG FALLBACK: Original intent: '{inferred_method}'")
                        print(f"🔍 DEBUG FALLBACK: Config explanation: '{config.explanation}'")
                        
                        # CRITICAL: Remap to available explanation method for this experimental condition
                        # Force all explanation requests to use the method enabled for this condition
                        if config.explanation == 'counterfactual':
                            # Conditions 3-4: Only counterfactual (DiCE) available
                            # Map ANYTHING except anchor to DiCE
                            if inferred_method != 'anchor':
                                print(f"🔍 DEBUG FALLBACK: Remapping '{inferred_method}' → 'dice' for counterfactual condition")
                                inferred_method = 'dice'
                        elif config.explanation == 'feature_importance':
                            # Conditions 5-6: Only feature importance (SHAP) available
                            # Map ANYTHING except anchor to SHAP
                            if inferred_method != 'anchor':
                                print(f"🔍 DEBUG FALLBACK: Remapping '{inferred_method}' → 'shap' for feature_importance condition")
                                inferred_method = 'shap'
                        
                        print(f"🔍 DEBUG FALLBACK: Final method after remapping: '{inferred_method}'")
                        
                        # Update the intent in the result
                        intent_result_fallback['intent'] = inferred_method
                        
                        # Route directly to XAI method with remapped intent
                        from xai_methods import route_to_xai_method
                        explanation_result = route_to_xai_method(self.agent, intent_result_fallback)
                        explanation = explanation_result.get('explanation', 'Sorry, I could not generate an explanation.')
                        
                        # Store SHAP results if applicable
                        if (inferred_method == 'shap' and 
                            isinstance(explanation_result, dict) and 
                            ('feature_impacts' in explanation_result or 'shap_values' in explanation_result)):
                            self.last_shap_result = explanation_result
                        else:
                            self.last_shap_result = None
                    else:
                        # Truly couldn't classify - use original fallback
                        explanation = self.agent.handle_user_input(user_input)
                except Exception as e:
                    print(f"🔍 DEBUG FALLBACK ERROR: {str(e)}")
                    explanation = self.agent.handle_user_input(user_input)
            
            # Format the explanation nicely
            if isinstance(explanation, dict) and 'explanation' in explanation:
                formatted_explanation = explanation['explanation']
            elif isinstance(explanation, str):
                formatted_explanation = explanation
            else:
                formatted_explanation = str(explanation)

            # XAI methods already enhance with LLM - no need to enhance again
            # Double enhancement causes wrapper text and duplicate content
            # For LOW anthropomorphism, return explanation without header (already has its own structure)
            if not config.show_anthropomorphic:
                return formatted_explanation
            else:
                return f"**Explanation:**\n\n{formatted_explanation}"
            
        except Exception as e:
            return ("I'm sorry, I couldn't generate that explanation right now. "
                   "Would you like to try asking differently or start a new application? "
                   f"Error: {str(e)}")

    def _is_xai_query(self, text: str) -> bool:
        """Decide if user input is an XAI question using NLU intent mapping with a small fallback.
        Uses sentence-transformers-based classifier; falls back to lightweight keywords if needed."""
        try:
            intent_result, _, _ = self.agent.nlu_model.classify_intent(text)
            if isinstance(intent_result, dict):
                return intent_result.get('intent') in {'shap', 'dice', 'anchor'}
        except Exception:
            pass
        # Fallback minimal heuristic
        t = (text or '').lower()
        return any(k in t for k in ['why', 'explain', 'factor', 'feature', 'importance', 'what if', 'change', 'counterfactual', 'rule', 'anchor'])

    def _process_field_input(self, field: str, user_input: str) -> Dict[str, Any]:
        """Process input for a specific field"""
        try:
            # Check if user is asking a meta-question (why/what/how) before validation
            from ab_config import config
            if NATURAL_CONVERSATION_AVAILABLE:
                meta_response = handle_meta_question(field, user_input, high_anthropomorphism=config.show_anthropomorphic)
                if meta_response:
                    # User is asking about the process, not providing data
                    return {
                        'success': False,
                        'message': meta_response
                    }
            
            # Extract value from user input
            value = self._extract_field_value(field, user_input)
            
            if value is None:
                self.field_attempts[field] = self.field_attempts.get(field, 0) + 1
                if self.field_attempts[field] >= 3:
                    return {
                        'success': False,
                        'message': f"I'm having trouble understanding your {field.replace('_', ' ')}. Let me provide some specific examples to help:\n\n{self._get_field_help(field)}\n\nOr you can say 'help' for more guidance."
                    }
                
                # Provide context-specific help on second attempt
                help_msg = self._get_smart_validation_message(field, user_input, self.field_attempts[field])
                return {
                    'success': False,
                    'message': help_msg
                }
            
            # Validate the value
            validation_result = self._validate_field_value(field, value)
            if not validation_result['valid']:
                self.field_attempts[field] = self.field_attempts.get(field, 0) + 1
                
                # Use smart validation message instead of generic one
                if self.field_attempts[field] >= 3:
                    err = f"I'm having trouble understanding your {field.replace('_', ' ')}. Let me provide some specific examples to help:\n\n{self._get_field_help(field)}\n\nOr type **'?'** to see all valid options you can copy-paste!"
                else:
                    err = self._get_smart_validation_message(field, user_input, self.field_attempts[field])
                
                return {
                    'success': False,
                    'message': err
                }
            
            # Set the normalized value if provided
            normalized = validation_result.get('normalized', value)
            setattr(self.application, field, normalized)
            self.field_attempts[field] = 0  # Reset attempts on success
            
            warn = validation_result.get('warning')
            
            # Generate natural confirmation message using LLM
            from ab_config import config
            if config.show_anthropomorphic:
                base_msg = f"Got it! {field.replace('_', ' ').title()}: {normalized}"
            else:
                base_msg = f"Confirmed. {field.replace('_', ' ').title()}: {normalized}"
            
            # Enhance with LLM for natural conversation
            if NATURAL_CONVERSATION_AVAILABLE:
                try:
                    context = {'field': field, 'value': normalized}
                    enhanced = enhance_response(
                        base_msg,
                        context,
                        "field_confirmation",
                        high_anthropomorphism=config.show_anthropomorphic
                    )
                    if enhanced and len(enhanced.strip()) > 5:
                        base_msg = enhanced
                except Exception:
                    pass  # Keep base message
            
            if warn:
                base_msg += f"\n\n⚠️ {self._format_warning(field, warn)}"
            
            return {
                'success': True,
                'message': base_msg
            }
            
        except Exception as e:
            return {
                'success': False,
                'message': "Sorry, there was an error processing that. Please try again."
            }

    def _extract_field_value(self, field: str, user_input: str):
        """Extract field value from user input using pattern matching"""
        user_input = user_input.strip()
        
        # Numeric fields
        if field in ['age', 'hours_per_week', 'capital_gain', 'capital_loss', 'education_num']:
            # Look for numbers in the input
            numbers = re.findall(r'\d+', user_input)
            if numbers:
                return int(numbers[0])
            return None
        
        # For categorical fields, try to match against known values
        elif field == 'sex':
            if re.search(r'\b(male|man|m)\b', user_input, re.IGNORECASE):
                return 'Male'
            elif re.search(r'\b(female|woman|f)\b', user_input, re.IGNORECASE):
                return 'Female'
        
        elif field == 'marital_status':
            input_lower = user_input.lower()
            if 'married' in input_lower and 'spouse' in input_lower:
                return 'Married-civ-spouse'
            elif 'married' in input_lower:
                return 'Married-civ-spouse'
            elif 'divorced' in input_lower:
                return 'Divorced'
            elif 'never' in input_lower or 'single' in input_lower:
                return 'Never-married'
            elif 'separated' in input_lower:
                return 'Separated'
            elif 'widow' in input_lower:
                return 'Widowed'
        
        # Handle "Other" responses for any categorical field
        if user_input.lower() in ['other', 'others', 'something else', 'none of the above']:
            return 'Other'
        
        # For other fields, return the input as-is for now
        # This can be enhanced with more sophisticated matching
        return user_input if user_input else None

    def _validate_field_value(self, field: str, value) -> Dict[str, Any]:
        """Validate field value against rules"""
        if field in self.validation_rules:
            rules = self.validation_rules[field]
            
            if rules['type'] == 'int':
                try:
                    int_val = int(value)
                    if int_val < rules.get('min', float('-inf')) or int_val > rules.get('max', float('inf')):
                        return {
                            'valid': False,
                            'message': f"Please enter a number between {rules.get('min', 'any')} and {rules.get('max', 'any')} for {self._pretty_field(field)}."
                        }
                    # Soft plausibility warnings for extreme-but-allowed values
                    warning = None
                    if field == 'hours_per_week' and int_val >= 80:
                        warning = "That is an unusually high number of weekly hours. Please confirm this is intentional."
                    if field == 'age' and int_val >= 85:
                        warning = (warning or "") + (" " if warning else "") + "Age is at the extreme upper bound of the dataset."
                    if field == 'capital_gain' and int_val > 5000:
                        warning = (warning or "") + (" " if warning else "") + "Note: Capital gains above $5,000 are relatively uncommon in this assessment."
                    if field == 'capital_loss' and int_val > 2000:
                        warning = (warning or "") + (" " if warning else "") + "Capital loss values above $2,000 are less common."
                    result = {'valid': True, 'message': '', 'normalized': int_val}
                    if warning:
                        result['warning'] = warning
                    return result
                except ValueError:
                    return {
                        'valid': False,
                        'message': f"Please enter a valid number for {self._pretty_field(field)}."
                    }

        # Enforce categorical whitelists (case-insensitive) and normalize to canonical values
        if field in getattr(self, 'allowed_values', {}):
            allowed = self.allowed_values[field]
            # Exact match
            if value in allowed:
                return {'valid': True, 'message': '', 'normalized': value}
            # Case-insensitive match
            val_norm = str(value).strip()
            for opt in allowed:
                if val_norm.lower() == opt.lower():
                    return {'valid': True, 'message': '', 'normalized': opt}
            
            # === COMPREHENSIVE FUZZY MATCHING FOR ALL CATEGORICAL FIELDS ===
            print(f"🔍 DEBUG: Fuzzy matching for field='{field}', val_norm='{val_norm}'")
            
            # Workclass: Employment type mappings
            if field == 'workclass':
                workclass_mapping = {
                    # Private sector variations
                    'private': 'Private', 'private sector': 'Private', 'pvt': 'Private',
                    'private company': 'Private', 'private employer': 'Private', 'pvt sector': 'Private',
                    'corporate': 'Private', 'company': 'Private', 'business': 'Private',
                    
                    # Self-employed not incorporated
                    'self employed': 'Self-emp-not-inc', 'self-employed': 'Self-emp-not-inc',
                    'self emp': 'Self-emp-not-inc', 'self employed not incorporated': 'Self-emp-not-inc',
                    'freelance': 'Self-emp-not-inc', 'freelancer': 'Self-emp-not-inc',
                    'independent': 'Self-emp-not-inc', 'contractor': 'Self-emp-not-inc',
                    'self': 'Self-emp-not-inc', 'own business': 'Self-emp-not-inc',
                    
                    # Self-employed incorporated
                    'self employed incorporated': 'Self-emp-inc', 'self-employed inc': 'Self-emp-inc',
                    'self emp inc': 'Self-emp-inc', 'incorporated': 'Self-emp-inc',
                    'own company': 'Self-emp-inc', 'business owner': 'Self-emp-inc',
                    
                    # Federal government
                    'federal': 'Federal-gov', 'federal government': 'Federal-gov',
                    'fed gov': 'Federal-gov', 'federal govt': 'Federal-gov',
                    'fed': 'Federal-gov', 'us government': 'Federal-gov',
                    'national government': 'Federal-gov',
                    
                    # Local government
                    'local': 'Local-gov', 'local government': 'Local-gov',
                    'local gov': 'Local-gov', 'local govt': 'Local-gov',
                    'city': 'Local-gov', 'municipal': 'Local-gov',
                    'county': 'Local-gov', 'city government': 'Local-gov',
                    
                    # State government
                    'state': 'State-gov', 'state government': 'State-gov',
                    'state gov': 'State-gov', 'state govt': 'State-gov',
                    
                    # Government (general - default to Federal)
                    'government': 'Federal-gov', 'govt': 'Federal-gov', 'gov': 'Federal-gov',
                    'public sector': 'Federal-gov', 'public': 'Federal-gov',
                    
                    # Other
                    'without pay': 'Without-pay', 'unpaid': 'Without-pay', 'volunteer': 'Without-pay',
                    'never worked': 'Never-worked', 'never': 'Never-worked', 'unemployed': 'Never-worked',
                    'unknown': '?', 'not sure': '?', 'prefer not to say': '?',
                }
                
                val_lower = val_norm.lower()
                if val_lower in workclass_mapping:
                    return {'valid': True, 'message': '', 'normalized': workclass_mapping[val_lower]}
            
            # Education: Education level mappings
            if field == 'education':
                education_mapping = {
                    # Doctorate variations
                    'phd': 'Doctorate', 'doctorate': 'Doctorate', 'doctoral': 'Doctorate',
                    'ph.d': 'Doctorate', 'ph.d.': 'Doctorate', 'doctor': 'Doctorate',
                    'doctoral degree': 'Doctorate',
                    
                    # Professional school
                    'professional': 'Prof-school', 'prof school': 'Prof-school',
                    'professional school': 'Prof-school', 'professional degree': 'Prof-school',
                    'law school': 'Prof-school', 'medical school': 'Prof-school',
                    'mba': 'Prof-school', 'jd': 'Prof-school', 'md': 'Prof-school',
                    
                    # Masters variations
                    'masters': 'Masters', 'master': 'Masters', "master's": 'Masters',
                    'masters degree': 'Masters', 'graduate degree': 'Masters',
                    'ms': 'Masters', 'm.s.': 'Masters', 'ma': 'Masters', 'm.a.': 'Masters',
                    'msc': 'Masters', 'grad school': 'Masters',
                    
                    # Bachelors variations
                    'bachelors': 'Bachelors', 'bachelor': 'Bachelors', "bachelor's": 'Bachelors',
                    'bachelors degree': 'Bachelors', 'undergraduate': 'Bachelors',
                    'college degree': 'Bachelors', 'university degree': 'Bachelors',
                    'bs': 'Bachelors', 'b.s.': 'Bachelors', 'ba': 'Bachelors', 'b.a.': 'Bachelors',
                    '4 year degree': 'Bachelors', 'four year degree': 'Bachelors',
                    
                    # Associates vocational
                    'associate vocational': 'Assoc-voc', 'assoc voc': 'Assoc-voc',
                    'associates voc': 'Assoc-voc', 'vocational': 'Assoc-voc',
                    'trade school': 'Assoc-voc', 'technical school': 'Assoc-voc',
                    
                    # Associates academic
                    'associate': 'Assoc-acdm', 'associates': 'Assoc-acdm',
                    'associate degree': 'Assoc-acdm', 'associates degree': 'Assoc-acdm',
                    'assoc': 'Assoc-acdm', 'aa': 'Assoc-acdm', 'as': 'Assoc-acdm',
                    '2 year degree': 'Assoc-acdm', 'two year degree': 'Assoc-acdm',
                    'community college': 'Assoc-acdm',
                    
                    # Some college
                    'some college': 'Some-college', 'college': 'Some-college',
                    'some university': 'Some-college', 'incomplete college': 'Some-college',
                    'partial college': 'Some-college', 'attended college': 'Some-college',
                    
                    # High school graduate
                    'high school': 'HS-grad', 'hs grad': 'HS-grad', 'hs-grad': 'HS-grad',
                    'high school graduate': 'HS-grad', 'high school diploma': 'HS-grad',
                    'hs diploma': 'HS-grad', 'diploma': 'HS-grad', 'graduated high school': 'HS-grad',
                    'secondary school': 'HS-grad', 'secondary': 'HS-grad',
                    
                    # 12th grade
                    '12th': '12th', '12th grade': '12th', 'grade 12': '12th',
                    'twelfth grade': '12th', 'senior year': '12th',
                    
                    # 11th grade
                    '11th': '11th', '11th grade': '11th', 'grade 11': '11th',
                    'eleventh grade': '11th', 'junior year': '11th',
                    
                    # 10th grade
                    '10th': '10th', '10th grade': '10th', 'grade 10': '10th',
                    'tenth grade': '10th', 'sophomore year': '10th',
                    
                    # 9th grade
                    '9th': '9th', '9th grade': '9th', 'grade 9': '9th',
                    'ninth grade': '9th', 'freshman year': '9th',
                    
                    # Elementary grades
                    '7th-8th': '7th-8th', '7th 8th': '7th-8th', 'middle school': '7th-8th',
                    '5th-6th': '5th-6th', '5th 6th': '5th-6th', 'elementary': '5th-6th',
                    '1st-4th': '1st-4th', '1st 4th': '1st-4th', 'primary school': '1st-4th',
                    'preschool': 'Preschool', 'pre school': 'Preschool', 'kindergarten': 'Preschool',
                }
                
                val_lower = val_norm.lower()
                print(f"🎓 DEBUG: Education check - val_lower='{val_lower}', in mapping: {val_lower in education_mapping}")
                if val_lower in education_mapping:
                    print(f"✅ DEBUG: Education match found: '{val_lower}' → '{education_mapping[val_lower]}'")
                    return {'valid': True, 'message': '', 'normalized': education_mapping[val_lower]}
            
            # Marital Status: Relationship status mappings
            if field == 'marital_status':
                marital_mapping = {
                    # Married civilian spouse
                    'married': 'Married-civ-spouse', 'married civilian': 'Married-civ-spouse',
                    'married civ spouse': 'Married-civ-spouse', 'spouse': 'Married-civ-spouse',
                    'wed': 'Married-civ-spouse', 'wedded': 'Married-civ-spouse',
                    
                    # Divorced
                    'divorced': 'Divorced', 'divorce': 'Divorced', 'div': 'Divorced',
                    'ex spouse': 'Divorced', 'former spouse': 'Divorced',
                    
                    # Never married
                    'never married': 'Never-married', 'single': 'Never-married',
                    'unmarried': 'Never-married', 'never wed': 'Never-married',
                    'not married': 'Never-married', 'bachelor': 'Never-married',
                    'bachelorette': 'Never-married',
                    
                    # Separated
                    'separated': 'Separated', 'sep': 'Separated', 'legal separation': 'Separated',
                    'legally separated': 'Separated',
                    
                    # Widowed
                    'widowed': 'Widowed', 'widow': 'Widowed', 'widower': 'Widowed',
                    
                    # Married spouse absent
                    'married spouse absent': 'Married-spouse-absent',
                    'spouse absent': 'Married-spouse-absent', 'separated married': 'Married-spouse-absent',
                    
                    # Married armed forces spouse
                    'married af spouse': 'Married-AF-spouse', 'military spouse': 'Married-AF-spouse',
                    'armed forces spouse': 'Married-AF-spouse', 'military': 'Married-AF-spouse',
                }
                
                val_lower = val_norm.lower()
                if val_lower in marital_mapping:
                    return {'valid': True, 'message': '', 'normalized': marital_mapping[val_lower]}
            
            # Occupation: Job type mappings
            if field == 'occupation':
                occupation_mapping = {
                    # Tech support
                    'tech support': 'Tech-support', 'tech': 'Tech-support', 'it': 'Tech-support',
                    'technical support': 'Tech-support', 'help desk': 'Tech-support',
                    'it support': 'Tech-support', 'computer support': 'Tech-support',
                    'technology': 'Tech-support',
                    
                    # Craft repair
                    'craft': 'Craft-repair', 'repair': 'Craft-repair', 'craft repair': 'Craft-repair',
                    'mechanic': 'Craft-repair', 'electrician': 'Craft-repair',
                    'plumber': 'Craft-repair', 'carpenter': 'Craft-repair',
                    'technician': 'Craft-repair', 'maintenance': 'Craft-repair',
                    
                    # Other service
                    'service': 'Other-service', 'other service': 'Other-service',
                    'customer service': 'Other-service', 'hospitality': 'Other-service',
                    'waiter': 'Other-service', 'waitress': 'Other-service',
                    'server': 'Other-service', 'food service': 'Other-service',
                    
                    # Sales
                    'sales': 'Sales', 'sale': 'Sales', 'salesperson': 'Sales',
                    'retail': 'Sales', 'cashier': 'Sales', 'sales rep': 'Sales',
                    'sales representative': 'Sales', 'seller': 'Sales',
                    
                    # Executive managerial
                    'manager': 'Exec-managerial', 'management': 'Exec-managerial',
                    'executive': 'Exec-managerial', 'exec': 'Exec-managerial',
                    'managerial': 'Exec-managerial', 'supervisor': 'Exec-managerial',
                    'director': 'Exec-managerial', 'ceo': 'Exec-managerial',
                    'president': 'Exec-managerial', 'vp': 'Exec-managerial',
                    'vice president': 'Exec-managerial', 'administrator': 'Exec-managerial',
                    'boss': 'Exec-managerial', 'leader': 'Exec-managerial',
                    
                    # Professional specialty
                    'professional': 'Prof-specialty', 'prof specialty': 'Prof-specialty',
                    'specialist': 'Prof-specialty', 'professional specialty': 'Prof-specialty',
                    'engineer': 'Prof-specialty', 'doctor': 'Prof-specialty',
                    'lawyer': 'Prof-specialty', 'accountant': 'Prof-specialty',
                    'scientist': 'Prof-specialty', 'researcher': 'Prof-specialty',
                    'teacher': 'Prof-specialty', 'professor': 'Prof-specialty',
                    'architect': 'Prof-specialty', 'analyst': 'Prof-specialty',
                    
                    # Handlers cleaners
                    'handler': 'Handlers-cleaners', 'cleaner': 'Handlers-cleaners',
                    'handlers cleaners': 'Handlers-cleaners', 'janitor': 'Handlers-cleaners',
                    'custodian': 'Handlers-cleaners', 'cleaning': 'Handlers-cleaners',
                    
                    # Machine operator inspector
                    'machine operator': 'Machine-op-inspct', 'operator': 'Machine-op-inspct',
                    'inspector': 'Machine-op-inspct', 'machine op': 'Machine-op-inspct',
                    'factory worker': 'Machine-op-inspct', 'assembly': 'Machine-op-inspct',
                    
                    # Administrative clerical
                    'admin': 'Adm-clerical', 'administrative': 'Adm-clerical',
                    'clerical': 'Adm-clerical', 'adm clerical': 'Adm-clerical',
                    'office': 'Adm-clerical', 'secretary': 'Adm-clerical',
                    'receptionist': 'Adm-clerical', 'clerk': 'Adm-clerical',
                    'office worker': 'Adm-clerical', 'assistant': 'Adm-clerical',
                    
                    # Farming fishing
                    'farming': 'Farming-fishing', 'fishing': 'Farming-fishing',
                    'farmer': 'Farming-fishing', 'agriculture': 'Farming-fishing',
                    'agricultural': 'Farming-fishing', 'farm': 'Farming-fishing',
                    'fisherman': 'Farming-fishing',
                    
                    # Transport moving
                    'transport': 'Transport-moving', 'transportation': 'Transport-moving',
                    'driver': 'Transport-moving', 'mover': 'Transport-moving',
                    'truck driver': 'Transport-moving', 'delivery': 'Transport-moving',
                    'courier': 'Transport-moving', 'logistics': 'Transport-moving',
                    
                    # Private house service
                    'private house': 'Priv-house-serv', 'house service': 'Priv-house-serv',
                    'domestic': 'Priv-house-serv', 'housekeeper': 'Priv-house-serv',
                    'maid': 'Priv-house-serv', 'nanny': 'Priv-house-serv',
                    
                    # Protective service
                    'protective': 'Protective-serv', 'protective service': 'Protective-serv',
                    'police': 'Protective-serv', 'security': 'Protective-serv',
                    'guard': 'Protective-serv', 'firefighter': 'Protective-serv',
                    'officer': 'Protective-serv', 'cop': 'Protective-serv',
                    
                    # Armed forces
                    'armed forces': 'Armed-Forces', 'military': 'Armed-Forces',
                    'army': 'Armed-Forces', 'navy': 'Armed-Forces',
                    'air force': 'Armed-Forces', 'marines': 'Armed-Forces',
                    'soldier': 'Armed-Forces', 'service member': 'Armed-Forces',
                    
                    # Unknown
                    'unknown': '?', 'not sure': '?', 'prefer not to say': '?',
                }
                
                val_lower = val_norm.lower()
                if val_lower in occupation_mapping:
                    return {'valid': True, 'message': '', 'normalized': occupation_mapping[val_lower]}
            
            # Sex: Gender mappings
            if field == 'sex':
                sex_mapping = {
                    # Male
                    'male': 'Male', 'm': 'Male', 'man': 'Male', 'boy': 'Male',
                    'guy': 'Male', 'gentleman': 'Male', 'he': 'Male',
                    
                    # Female
                    'female': 'Female', 'f': 'Female', 'woman': 'Female',
                    'girl': 'Female', 'lady': 'Female', 'she': 'Female',
                }
                
                val_lower = val_norm.lower()
                if val_lower in sex_mapping:
                    return {'valid': True, 'message': '', 'normalized': sex_mapping[val_lower]}
            
            # Race: Ethnicity mappings
            if field == 'race':
                race_mapping = {
                    # White
                    'white': 'White', 'caucasian': 'White', 'european': 'White',
                    
                    # Black
                    'black': 'Black', 'african american': 'Black', 'african': 'Black',
                    'afro american': 'Black',
                    
                    # Asian Pacific Islander
                    'asian': 'Asian-Pac-Islander', 'asian pacific islander': 'Asian-Pac-Islander',
                    'asian pac islander': 'Asian-Pac-Islander', 'pacific islander': 'Asian-Pac-Islander',
                    'filipino': 'Asian-Pac-Islander', 'chinese': 'Asian-Pac-Islander',
                    'japanese': 'Asian-Pac-Islander', 'korean': 'Asian-Pac-Islander',
                    'vietnamese': 'Asian-Pac-Islander', 'hawaiian': 'Asian-Pac-Islander',
                    
                    # American Indian Eskimo
                    'american indian': 'Amer-Indian-Eskimo', 'native american': 'Amer-Indian-Eskimo',
                    'indigenous': 'Amer-Indian-Eskimo', 'eskimo': 'Amer-Indian-Eskimo',
                    'native': 'Amer-Indian-Eskimo', 'indian': 'Amer-Indian-Eskimo',
                    
                    # Other
                    'other': 'Other', 'mixed': 'Other', 'multiracial': 'Other',
                    'hispanic': 'Other', 'latino': 'Other', 'latina': 'Other',
                }
                
                val_lower = val_norm.lower()
                if val_lower in race_mapping:
                    return {'valid': True, 'message': '', 'normalized': race_mapping[val_lower]}
            
            # Relationship: Family relationship mappings
            if field == 'relationship':
                relationship_mapping = {
                    # Wife
                    'wife': 'Wife', 'married woman': 'Wife', 'spouse female': 'Wife',
                    
                    # Husband
                    'husband': 'Husband', 'married man': 'Husband', 'spouse male': 'Husband',
                    
                    # Own child
                    'own child': 'Own-child', 'child': 'Own-child', 'son': 'Own-child',
                    'daughter': 'Own-child', 'kid': 'Own-child', 'offspring': 'Own-child',
                    
                    # Not in family
                    'not in family': 'Not-in-family', 'alone': 'Not-in-family',
                    'independent': 'Not-in-family', 'single person': 'Not-in-family',
                    'living alone': 'Not-in-family',
                    
                    # Other relative
                    'other relative': 'Other-relative', 'relative': 'Other-relative',
                    'cousin': 'Other-relative', 'aunt': 'Other-relative',
                    'uncle': 'Other-relative', 'niece': 'Other-relative',
                    'nephew': 'Other-relative', 'grandparent': 'Other-relative',
                    'grandchild': 'Other-relative',
                    
                    # Unmarried
                    'unmarried': 'Unmarried', 'partner': 'Unmarried',
                    'boyfriend': 'Unmarried', 'girlfriend': 'Unmarried',
                    'domestic partner': 'Unmarried', 'roommate': 'Unmarried',
                }
                
                val_lower = val_norm.lower()
                if val_lower in relationship_mapping:
                    return {'valid': True, 'message': '', 'normalized': relationship_mapping[val_lower]}
            
            # Native Country: Country name mappings
            if field == 'native_country':
                country_mapping = {
                    # United States variations
                    'united states': 'United-States', 'usa': 'United-States', 'us': 'United-States',
                    'u.s.': 'United-States', 'u.s.a.': 'United-States', 'america': 'United-States',
                    'united states of america': 'United-States', 'the us': 'United-States',
                    
                    # Country spelling variations
                    'colombia': 'Columbia', 'columbia': 'Columbia',
                    'netherlands': 'Holand-Netherlands', 'holland': 'Holand-Netherlands',
                    'hong kong': 'Hong', 'hongkong': 'Hong',
                    'south korea': 'South', 'korea': 'South', 's korea': 'South',
                    'trinidad': 'Trinadad&Tobago', 'trinidad and tobago': 'Trinadad&Tobago',
                    'el salvador': 'El-Salvador', 'salvador': 'El-Salvador',
                    'dominican republic': 'Dominican-Republic', 'dominican': 'Dominican-Republic',
                    'puerto rico': 'Puerto-Rico', 'puertorico': 'Puerto-Rico',
                    'us territory': 'Outlying-US(Guam-USVI-etc)', 'guam': 'Outlying-US(Guam-USVI-etc)',
                    'virgin islands': 'Outlying-US(Guam-USVI-etc)', 'usvi': 'Outlying-US(Guam-USVI-etc)',
                    'former yugoslavia': 'Yugoslavia', 'yugoslavia': 'Yugoslavia',
                    
                    # Full country names
                    'cambodia': 'Cambodia', 'canada': 'Canada', 'china': 'China',
                    'cuba': 'Cuba', 'ecuador': 'Ecuador', 'england': 'England',
                    'france': 'France', 'germany': 'Germany', 'greece': 'Greece',
                    'guatemala': 'Guatemala', 'haiti': 'Haiti', 'honduras': 'Honduras',
                    'hungary': 'Hungary', 'india': 'India', 'iran': 'Iran',
                    'ireland': 'Ireland', 'italy': 'Italy', 'jamaica': 'Jamaica',
                    'japan': 'Japan', 'laos': 'Laos', 'mexico': 'Mexico',
                    'nicaragua': 'Nicaragua', 'peru': 'Peru', 'philippines': 'Philippines',
                    'poland': 'Poland', 'portugal': 'Portugal', 'scotland': 'Scotland',
                    'taiwan': 'Taiwan', 'thailand': 'Thailand', 'vietnam': 'Vietnam',
                    
                    # Unknown
                    'unknown': '?', 'not sure': '?', 'prefer not to say': '?',
                }
                
                val_lower = val_norm.lower()
                if val_lower in country_mapping:
                    return {'valid': True, 'message': '', 'normalized': country_mapping[val_lower]}
                
                # Try partial match for countries
                for friendly, technical in country_mapping.items():
                    if friendly in val_lower or val_lower in friendly:
                        return {'valid': True, 'message': '', 'normalized': technical}
            # === FALLBACK: FUZZY MATCHING WITH DIFFLIB ===
            # If no exact mapping found, try fuzzy matching against allowed values
            suggestions = difflib.get_close_matches(val_norm, allowed, n=1, cutoff=0.7)
            if suggestions:
                # Found a close match - auto-normalize to it
                return {'valid': True, 'message': '', 'normalized': suggestions[0]}
            
            # Case-insensitive fuzzy match (try lowercase comparison)
            suggestions_lower = difflib.get_close_matches(
                val_norm.lower(), 
                [opt.lower() for opt in allowed], 
                n=1, 
                cutoff=0.7
            )
            if suggestions_lower:
                # Find the original allowed value
                for opt in allowed:
                    if opt.lower() == suggestions_lower[0]:
                        return {'valid': True, 'message': '', 'normalized': opt}
            
            # Special handling for generic 'other'
            if val_norm.lower() in ['other', 'others', 'something else', 'none of the above']:
                if field == 'race' and 'Other' in allowed:
                    return {'valid': True, 'message': '', 'normalized': 'Other'}
                if field == 'occupation':
                    return {
                        'valid': False,
                        'message': "For occupation, the dataset has 'Other-service' but not plain 'Other'. Please choose 'Other-service' or a specific occupation."
                    }
                if field == 'relationship':
                    return {
                        'valid': False,
                        'message': "For relationship, the dataset has 'Other-relative' but not plain 'Other'. Please choose 'Other-relative' or a specific relationship."
                    }
            
            # Unknown '?' permitted only if present in allowed list
            if val_norm == '?' and '?' in allowed:
                return {'valid': True, 'message': '', 'normalized': '?'}
            
            # No match found - provide helpful suggestions
            close_matches = difflib.get_close_matches(val_norm, allowed, n=3, cutoff=0.5)
            if close_matches:
                return {
                    'valid': False,
                    'message': f"'{value}' isn't a valid {self._pretty_field(field)}. Did you mean: {', '.join(close_matches)}?",
                    'allowed': allowed
                }
            
            return {
                'valid': False,
                'message': f"'{value}' isn't a valid {self._pretty_field(field)}.",
                'allowed': allowed
            }
        
        # Handle "Other" category and "?" (unknown/missing) values with accurate dataset information
        if value == 'Other' or value == '?' or value == 'Unknown/Prefer not to say':
            # Fields that actually DO have "Other" categories in the Adult dataset
            fields_with_other = ['race']  # race has "Other", occupation has "Other-service", relationship has "Other-relative"
            
            # Fields that support "?" (unknown/missing values)
            fields_with_unknown = ['workclass', 'occupation', 'native_country']
            
            # Fields that do NOT have "Other" or "?" categories
            fields_without_other_or_unknown = ['education', 'marital_status', 'sex', 'relationship']
            
            # Handle "?" values for fields that support them
            if value in ['?', 'Unknown/Prefer not to say'] and field in fields_with_unknown:
                return {'valid': True, 'message': ''}  # Accept "?" for these fields
            
            # Handle "Other" requests for fields that don't support either
            if field in fields_without_other_or_unknown:
                if field == 'sex':
                    return {
                        'valid': False,
                        'message': f"I understand that gender identity is more diverse than the binary options available. Unfortunately, the Adult (Census Income) dataset that trained this model only contains 'Male' and 'Female' categories. We are actively working on accommodating more inclusive gender categories in future versions. For now, could you please select the option that best represents you, or choose the one you're most comfortable with for this application?"
                    }
                else:
                    if value in ['?', 'Unknown/Prefer not to say']:
                        return {
                            'valid': False,
                            'message': f"I understand you may prefer not to specify, but this field doesn't support 'unknown' values in the dataset. Could you select the closest option for {self._pretty_field(field)}?"
                        }
                    else:
                        return {
                            'valid': False,
                            'message': f"The dataset doesn't include 'Other' as a category for {self._pretty_field(field)}. Please choose from the available options."
                        }
            elif field == 'occupation':
                # Occupation has "Other-service" but not plain "Other"
                return {
                    'valid': False,
                    'message': "For occupation, please choose 'Other-service' or a specific occupation from the list."
                }
            elif field == 'relationship':
                # Relationship has "Other-relative" but not plain "Other"  
                return {
                    'valid': False,
                    'message': "For relationship, please choose 'Other-relative' or one of the listed relationships."
                }
        
        return {'valid': True, 'message': '', 'normalized': value}

    def _setup_agent_instance(self):
        """Set up the agent's current instance using the user's application data"""
        try:
            # Convert application to the format expected by the agent
            app_data = self.application.to_dict()
            
            # Add missing required fields with defaults
            if 'fnlwgt' not in app_data or app_data['fnlwgt'] is None:
                app_data['fnlwgt'] = 100000
            
            # Create a DataFrame row representing this user's data
            app_df = pd.DataFrame([app_data])
            
            # Preprocess it the same way the training data was preprocessed
            app_df['income'] = '>50K' if self.application.loan_approved else '<=50K'
            from preprocessing import preprocess_adult
            app_processed = preprocess_adult(app_df)
            
            # Set this as the agent's current instance
            self.agent.current_instance = app_processed.drop('income', axis=1).iloc[0]
            self.agent.predicted_class = app_processed['income'].iloc[0]
            
            # CRITICAL: Pass the actual loan decision to agent for XAI explanations
            self.agent.loan_approved = self.application.loan_approved
            
        except Exception as e:
            # Fallback to random instance if user data setup fails
            print(f"Warning: Could not set up user instance, using random: {e}")
            self.agent.select_random_instance()

    def _get_next_question(self) -> str:
        """Get the next question to ask based on missing fields"""
        for field in self.field_order:
            if getattr(self.application, field) is None:
                self.current_field = field
                return self.field_prompts.get(field, f"Please provide your {field.replace('_', ' ')}:")
        
        self.current_field = None
        return ""

    def _show_progress(self) -> str:
        """Show current application progress with step information"""
        completion = self.application.calculate_completion()
        completed_fields = []
        missing_fields = []
        current_step = None
        
        for field in self.field_order:
            value = getattr(self.application, field)
            step_num = self.field_to_step.get(field, 0)
            step_desc = self.step_descriptions.get(step_num, field.replace('_', ' ').title())
            
            if value is not None:
                completed_fields.append(f"✅ **Step {step_num}:** {step_desc} - {value}")
            else:
                if current_step is None:  # First missing field is current step
                    current_step = step_num
                    missing_fields.append(f"🔄 **Step {step_num}:** {step_desc} *(Currently collecting)*")
                else:
                    missing_fields.append(f"⏳ **Step {step_num}:** {step_desc}")
        
        progress_msg = f"📊 **Application Progress: {completion:.0f}% Complete (Step {current_step or 10}/10)**\n\n"
        
        if completed_fields:
            progress_msg += "**✅ Completed Steps:**\n" + "\n".join(completed_fields) + "\n\n"
        
        if missing_fields:
            progress_msg += "**📋 Remaining Steps:**\n" + "\n".join(missing_fields) + "\n\n"
        
        if current_step:
            progress_msg += f"💡 **Next:** Continue with Step {current_step}\n\n"
        
        progress_msg += "Would you like to continue filling out the application?"
        return progress_msg

    def _show_application_summary(self) -> str:
        """Show complete application summary"""
        summary = "**Application Summary:**\n\n"
        
        for field in self.field_order:
            value = getattr(self.application, field)
            if value is not None:
                # Get friendly field name
                friendly_field = field.replace('_', ' ').title()
                
                # Format value based on field type
                if field in ['capital_gain', 'capital_loss']:
                    friendly_value = f"${value:,}" if isinstance(value, (int, float)) else str(value)
                elif field == 'hours_per_week':
                    friendly_value = f"{value} hours/week"
                elif field == 'age':
                    friendly_value = f"{value} years old"
                elif field in ['workclass', 'education', 'marital_status', 'occupation', 'relationship', 'race', 'native_country']:
                    # For categorical values, try to get friendly name from FEATURE_DISPLAY_NAMES
                    friendly_value = get_friendly_feature_name(f"{field}_{value}")
                    # If no mapping found, use original value
                    if friendly_value.startswith(field.title()) or friendly_value == f"{field}_{value}":
                        friendly_value = str(value)
                else:
                    friendly_value = str(value)
                
                summary += f"• {friendly_field}: {friendly_value}\n"
        
        return summary

    def _process_application(self) -> str:
        """Process the loan application using the ML model"""
        try:
            # Convert application to DataFrame for model prediction
            app_data = self.application.to_dict()
            app_df = pd.DataFrame([app_data])
            
            # Add a dummy income column for preprocessing (required by preprocess_adult)
            app_df['income'] = '>50K'  # Dummy value, will be ignored
            
            # Ensure all required columns are present for preprocessing
            required_columns = ['age', 'workclass', 'fnlwgt', 'education', 'education_num', 
                              'marital_status', 'occupation', 'relationship', 'race', 'sex',
                              'capital_gain', 'capital_loss', 'hours_per_week', 'native_country']
            
            # Add missing columns with sensible defaults
            if 'fnlwgt' not in app_df.columns:
                app_df['fnlwgt'] = 100000  # Default final weight
            
            if 'education_num' not in app_df.columns:
                # Map education level to education_num
                education_mapping = {
                    'Preschool': 1, '1st-4th': 2, '5th-6th': 3, '7th-8th': 4, '9th': 5,
                    '10th': 6, '11th': 7, '12th': 8, 'HS-grad': 9, 'Some-college': 10,
                    'Assoc-voc': 11, 'Assoc-acdm': 12, 'Bachelors': 13, 'Masters': 14,
                    'Prof-school': 15, 'Doctorate': 16
                }
                education_value = app_df['education'].iloc[0] if 'education' in app_df.columns else 'HS-grad'
                app_df['education_num'] = education_mapping.get(education_value, 9)
            
            # Now preprocess the data to match the model's expected format
            from preprocessing import preprocess_adult
            app_df_processed = preprocess_adult(app_df)
            
            # Remove the dummy income column before prediction
            X_features = app_df_processed.drop('income', axis=1)
            
            # Check for and handle any remaining NaN values (failsafe)
            if X_features.isnull().any().any():
                X_features = X_features.fillna(0)
            
            # Ensure all values are finite (no inf values)
            if not np.isfinite(X_features.values).all():
                X_features = X_features.replace([np.inf, -np.inf], 0)
            
            # Make prediction using the agent's model
            if self.agent.clf_display is not None:
                # Ensure feature alignment with training data
                # Get the expected features from training data
                if hasattr(self.agent, 'data') and 'X_display' in self.agent.data:
                    # Get training features for reference
                    train_df = pd.concat([self.agent.data['X_display'], self.agent.data['y_display']], axis=1)
                    train_df_processed = preprocess_adult(train_df)
                    expected_features = train_df_processed.drop('income', axis=1).columns.tolist()
                    
                    # Align prediction features with training features
                    for col in expected_features:
                        if col not in X_features.columns:
                            X_features[col] = 0  # Add missing columns with default value
                    
                    # Reorder columns to match training order
                    X_features = X_features[expected_features]
                
                prediction = self.agent.clf_display.predict(X_features)[0]
                
                # Map prediction to loan decision
                approved = prediction in ['>50K', '1', 'true', 'True', 'Approved']
                decision = "APPROVED" if approved else "NOT APPROVED"
                
                # Update the application with the loan decision
                self.application.loan_approved = approved
                
                self.conversation_state = ConversationState.COMPLETE
                
                result_msg = f"🎉 **Loan Application Result: {decision}**\n\n"
                
                if approved:
                    result_msg += ("Congratulations! Your loan application has been approved based on your profile. "
                                 "You should hear from our lending team within 2-3 business days.\n\n")
                else:
                    # Simple message for non-explanation conditions
                    if config.explanation == "none":
                        result_msg += ("Based on the data you provided, you could not qualify for the loan at this time. "
                                     "For further details, please visit your local branch.\n\n")
                    else:
                        result_msg += ("Unfortunately, your loan application was not approved at this time. "
                                     "This decision is based on various factors in your application.\n\n")
                
                # Only offer explanations if explanation mode is enabled
                if config.show_any_explanation:
                    result_msg += ("Would you like me to explain how this decision was made? "
                                 "Just ask me 'why' or 'explain the decision'.")
                else:
                    result_msg += "Would you like to start a new application?"
                
                return result_msg
            else:
                return "I'm sorry, there was an issue processing your application. Please try again later."
                
        except Exception as e:
            return f"Sorry, there was an error processing your application: {str(e)}"

    def _generate_explanation(self) -> str:
        """Generate explanation using existing XAI methods"""
        try:
            # Set current instance for explanation
            self.agent.current_instance = self.application.to_dict()
            self.agent.df_display_instance = pd.DataFrame([self.application.to_dict()])
            
            # Use existing XAI explanation
            explanation = self.agent.handle_user_input("Why was this decision made?")
            return f"**Decision Explanation:**\n\n{explanation}"
            
        except Exception as e:
            return "I'm sorry, I couldn't generate an explanation right now. The decision was based on various factors in your application profile."

    def _restart_application(self) -> str:
        """Restart the application process"""
        self.application = LoanApplication()
        self.conversation_state = ConversationState.GREETING
        self.current_field = None
        self.field_attempts = {}
        self.show_what_if_lab = False
        return "Great! Let's start a new loan application. Hi! I'm your loan application assistant. Would you like to start your loan application?"

    def get_conversation_state(self) -> Dict[str, Any]:
        """Get current conversation state for debugging/monitoring"""
        current_step = None
        if self.current_field and self.current_field in self.field_to_step:
            current_step = self.field_to_step[self.current_field]
        
        return {
            'state': self.conversation_state.value,
            'completion_percentage': self.application.calculate_completion(),
            'current_field': self.current_field,
            'current_step': current_step,
            'step_description': self.step_descriptions.get(current_step, '') if current_step else '',
            'is_complete': self.application.is_complete
        }
    
    def _get_progress_celebration(self, completion: float) -> str:
        """Return appropriate celebration message based on progress"""
        if config.show_anthropomorphic:
            # High anthropomorphism: Warm, encouraging
            if completion >= 75:
                return "We're almost done! 🎊"
            elif completion >= 50:
                return "Great progress! We're halfway there! 🚀"
            elif completion >= 25:
                return "Excellent! You're making great progress! 💪"
            else:
                return "Thank you so much! 😊"
        else:
            # Low anthropomorphism: Technical, factual
            if completion >= 75:
                return "Data collection 75% complete."
            elif completion >= 50:
                return "50% progress milestone reached."
            elif completion >= 25:
                return "25% data fields collected."
            else:
                return "Data received."
    
    def _get_success_message(self, field: str, normalized_value: str, completion: float, next_step: int = None) -> str:
        """Generate natural success message using LLM enhancement"""
        from ab_config import config
        
        # Build base message
        field_friendly = field.replace('_', ' ').title()
        celebration = self._get_progress_celebration(completion)
        
        if config.show_anthropomorphic:
            base_msg = f"Perfect! {celebration} I've recorded your {field_friendly}: {normalized_value}.\n\nProgress: {completion:.1f}% complete."
        else:
            base_msg = f"Data received. {field_friendly}: {normalized_value}. Progress: {completion:.1f}% complete."
        
        # Add next step info if available
        if next_step:
            step_desc = self.step_descriptions.get(next_step, '')
            if config.show_anthropomorphic:
                base_msg += f"\n\n📋 **Step {next_step}/10:** {step_desc}"
            else:
                base_msg += f"\n\nStep {next_step}/10: {step_desc}"
        
        # Enhance with LLM for natural conversation
        if NATURAL_CONVERSATION_AVAILABLE:
            try:
                context = {
                    'field': field,
                    'value': normalized_value,
                    'completion': completion,
                    'step': next_step
                }
                enhanced = enhance_response(
                    base_msg,
                    context,
                    "success_confirmation",
                    high_anthropomorphism=config.show_anthropomorphic
                )
                if enhanced and len(enhanced.strip()) > 10:
                    return enhanced
            except Exception:
                pass  # Fall back to base message
        
        return base_msg
    
    def _get_smart_validation_message(self, field: str, user_input: str, attempt: int) -> str:
        """Provide smart, context-aware validation messages"""
        from ab_config import config
        
        if config.show_anthropomorphic:
            # Try to get dynamic LLM-generated message first
            if NATURAL_CONVERSATION_AVAILABLE:
                try:
                    # Define expected format for each field
                    expected_formats = {
                        'age': "a number between 17-90, e.g., '25', '30', or '45'",
                        'workclass': "employment type (we support fuzzy matching! e.g., 'private', 'self-employed', 'government', etc. - or just '?' if unsure)",
                        'education': "education level (we support fuzzy matching! e.g., 'bachelor', 'phd', 'masters', 'high school', etc. - be flexible with user input, or '?' if unsure)",
                        'sex': "'Male' or 'Female'",
                        'marital_status': "one of: 'Married-civ-spouse', 'Never-married', 'Divorced', 'Separated', 'Widowed', 'Married-spouse-absent', 'Married-AF-spouse'",
                        'occupation': "a job category (or just '?' if you're not sure)",
                        'hours_per_week': "number of hours like '40', '35', or '50'",
                        'capital_gain': "amount like '5000' or '0' if none",
                        'capital_loss': "amount like '2000' or '0' if none",
                        'race': "one of: Black, Asian-Pac-Islander, Amer-Indian-Eskimo, White, or Other",
                        'native_country': "a country name from the 42 supported countries (or '?' if unsure)",
                        'relationship': "one of: Husband, Wife, Own-child, Not-in-family, Other-relative, Unmarried"
                    }
                    
                    expected_format = expected_formats.get(field, f"valid {field.replace('_', ' ')}")
                    llm_message = enhance_validation_message(field, user_input, expected_format, attempt, high_anthropomorphism=True)
                    
                    if llm_message:
                        return llm_message
                except Exception:
                    pass  # Fall back to hardcoded messages
            
            # Fallback: Hardcoded warm messages if LLM not available
            # High anthropomorphism: Warm, friendly, understanding
            field_examples = {
                'age': "I need a number for your age, please! 😊 For example: '25', '30', or '45'",
                'workclass': "No worries! Please choose from: Private sector, Self-employed, Federal/Local/State government, etc.\n\n💡 **Type '?' to see all valid options!**",
                'education': "I'd love to know your education level! Try: 'High school graduate', 'Bachelor's', 'Master's', 'Some college', etc.\n\n💡 **Type '?' to see all valid options you can copy-paste!** 📚",
                'sex': "Please let me know: 'Male' or 'Female' - these are the only categories in my dataset, I apologize for the limitation",
                'marital_status': "I need your marital status! Please choose: 'Married', 'Never married', 'Divorced', 'Separated', 'Widowed', etc.\n\n💡 **Type '?' for the full list!**",
                'occupation': "Tell me about your job! Please pick from the available categories.\n\n💡 **Type '?' to see all valid options you can copy-paste!** 💼",
                'hours_per_week': "How many hours do you work per week? Just give me a number like: '40', '35', or '50' ⏰",
                'capital_gain': "I need the amount of capital gains, or just '0' if you don't have any. For example: '5000' or '0'",
                'capital_loss': "Please tell me your capital losses, or '0' if none. For example: '2000' or '0'",
                'race': "Please help me understand your race/ethnicity.\n\n💡 **Type '?' to see all valid options!**",
                'native_country': "Which country are you from? I support all 42 countries in my training data! 🌍\n\n💡 **Type '?' to see the full list!**",
                'relationship': "What's your household relationship?\n\n💡 **Type '?' to see all valid options!**"
            }
            
            base_msg = f"Oops! I didn't quite catch that - '{user_input}' doesn't seem right for {field.replace('_', ' ')}. 🤔"
            help_msg = field_examples.get(field, f"Please share your {field.replace('_', ' ')} with me!")
            
            if attempt == 2:
                return f"{base_msg}\n\n💡 **Here's a tip:** {help_msg}\n\nLet's try again, or say 'help' if you need more options! 😊"
            else:
                return f"{base_msg}\n\n{help_msg}"
        else:
            # Low anthropomorphism: Technical, concise
            # Try to get dynamic LLM-generated message first
            if NATURAL_CONVERSATION_AVAILABLE:
                try:
                    # Define expected format for each field
                    expected_formats = {
                        'age': "a number between 17-90, e.g., '25', '30', or '45'",
                        'workclass': "employment type (fuzzy matching supported: e.g., 'private', 'self-employed', 'government', etc. - or '?' if unsure)",
                        'education': "education level (fuzzy matching supported: e.g., 'bachelor', 'phd', 'masters', 'high school', etc. - flexible input accepted, or '?' if unsure)",
                        'sex': "'Male' or 'Female'",
                        'marital_status': "one of: 'Married-civ-spouse', 'Never-married', 'Divorced', 'Separated', 'Widowed', 'Married-spouse-absent', 'Married-AF-spouse'",
                        'occupation': "a job category (or '?' if not sure)",
                        'hours_per_week': "number of hours like '40', '35', or '50'",
                        'capital_gain': "amount like '5000' or '0' if none",
                        'capital_loss': "amount like '2000' or '0' if none",
                        'race': "one of: Black, Asian-Pac-Islander, Amer-Indian-Eskimo, White, or Other",
                        'native_country': "a country name from the 42 supported countries (or '?' if unsure)",
                        'relationship': "one of: Husband, Wife, Own-child, Not-in-family, Other-relative, Unmarried"
                    }
                    
                    expected_format = expected_formats.get(field, f"valid {field.replace('_', ' ')}")
                    llm_message = enhance_validation_message(field, user_input, expected_format, attempt, high_anthropomorphism=False)
                    
                    if llm_message:
                        return llm_message
                except Exception:
                    pass  # Fall back to hardcoded messages
            
            # Fallback: Hardcoded technical messages if LLM not available
            field_examples = {
                'age': "I need a number for your age. For example: '25', '30', or '45'",
                'workclass': "Please choose from the available employment types.\n\nType '?' to see all valid options.",
                'education': "Please specify your education level.\n\nType '?' to see all valid options you can copy-paste.",
                'sex': "Please specify 'Male' or 'Female' - these are the only categories in the dataset I was trained on",
                'marital_status': "Please choose from the available marital status options.\n\nType '?' for the full list.",
                'occupation': "Please describe your job from the available categories.\n\nType '?' to see all valid options you can copy-paste.",
                'hours_per_week': "I need the number of hours you work per week. For example: '40', '35', or '50'",
                'capital_gain': "Enter the amount of capital gains, or '0' if none. For example: '5000' or '0'",
                'capital_loss': "Enter the amount of capital losses, or '0' if none. For example: '2000' or '0'",
                'race': "Please choose from the available race/ethnicity options.\n\nType '?' to see all valid options.",
                'native_country': "Please specify your country from the 42 supported countries.\n\nType '?' to see the full list.",
                'relationship': "Please choose from the available relationship options.\n\nType '?' for the full list."
            }
            
            base_msg = f"I didn't quite understand '{user_input}' for {field.replace('_', ' ')}."
            help_msg = field_examples.get(field, f"Please provide your {field.replace('_', ' ')}.")
            
            if attempt == 2:
                return f"{base_msg}\n\n💡 **Tip:** {help_msg}\n\nTry again, or say 'help' for more options."
            else:
                return f"{base_msg} {help_msg}"
    
    def _get_copyable_options(self, field: str) -> str:
        """Provide copyable list of valid options for categorical fields"""
        from ab_config import config
        
        if field not in self.allowed_values:
            # Numeric field - provide range info
            if field in self.validation_rules:
                rules = self.validation_rules[field]
                min_val = rules.get('min', 'any')
                max_val = rules.get('max', 'any')
                if config.show_anthropomorphic:
                    return f"📋 **{field.replace('_', ' ').title()}** is a number field.\n\nJust enter a value between **{min_val}** and **{max_val}**.\n\nFor example: `{(min_val + max_val) // 2 if isinstance(min_val, int) and isinstance(max_val, int) else '40'}`"
                else:
                    return f"**{field.replace('_', ' ').title()}:** Numeric field.\n\nRange: {min_val} to {max_val}\n\nExample: `{(min_val + max_val) // 2 if isinstance(min_val, int) and isinstance(max_val, int) else '40'}`"
        
        # Categorical field - provide full list
        options = self.allowed_values[field]
        
        if config.show_anthropomorphic:
            header = f"📋 **Here are all the valid options for {field.replace('_', ' ')}:**\n\n"
            header += "💡 **Tip:** You can copy-paste any of these exactly:\n\n"
        else:
            header = f"**Valid options for {field.replace('_', ' ')}:**\n\n"
            header += "Copy-paste one of these options:\n\n"
        
        # Format options in a clean, copyable list
        options_list = "\n".join([f"• `{opt}`" for opt in options])
        
        if config.show_anthropomorphic:
            footer = "\n\n✨ **Or just describe it naturally** - I can understand variations like 'private', 'bachelor', 'single', etc.!"
            footer += "\n\n👇 **Even easier:** Look for the **clickable quick-select buttons** right below this chat - just click your choice!"
        else:
            footer = "\n\nNote: Natural language variations (e.g., 'private', 'bachelor', 'single') are also accepted."
            footer += "\n\n**Quick-select buttons:** Clickable options are displayed below the chat for faster selection."
        
        return header + options_list + footer
    
    def _get_field_help(self, field: str) -> str:
        """Provide detailed help for specific fields"""
        help_texts = {
            'age': "**Age Examples:**\n• Just enter a number: 25, 30, 45\n• Must be between 17-90 years old",
            'workclass': "**Employment Type (9 categories):**\n• Private sector (most common)\n• Self-employed\n• Federal/Local/State government\n• Without pay, Never worked\n• '?' if unknown/missing",
            'education': "**Education Level (16 categories):**\n• High school graduate, Bachelor's, Master's, Doctorate\n• Some college, Associate's degree\n• 1st-4th grade, 5th-6th grade, 7th-8th grade\n• 9th, 10th, 11th, 12th grade\n• Preschool, Professional school",
            'sex': "**Gender (2 categories only):**\n• Male\n• Female\n\n⚠️ This dataset was created in 1994 and only includes these binary options. We acknowledge this limitation and are working toward more inclusive data.",
            'marital_status': "**Marital Status (7 categories):**\n• Married\n• Never married\n• Divorced, Separated, Widowed\n• Married but spouse absent\n• Married to armed forces spouse",
            'occupation': "**Occupation (15 categories):**\n• Professional, Management, Tech support\n• Sales, Craft/repair, Administrative\n• Service, Farming/fishing\n• Handlers/cleaners, Machine operators\n• Transportation, Protective services\n• Private household service, Armed Forces\n• '?' if unknown",
            'race': "**Race/Ethnicity (5 categories):**\n• White, Black\n• Asian-Pacific Islander\n• Indigenous American\n• Other ✅",
            'native_country': "**Native Country (42 countries!):**\n• United States, Canada, Mexico\n• England, Scotland, Germany, France, Italy, Greece, Ireland, Portugal, Poland, Hungary, Netherlands\n• Philippines, India, China, Japan, Taiwan, Thailand, Vietnam, Cambodia, Laos, Hong Kong, South Korea\n• Jamaica, Haiti, Cuba, Puerto Rico, Dominican Republic, Trinidad & Tobago, Guatemala, Honduras, El Salvador, Nicaragua, Ecuador, Peru, Colombia\n• Iran, Former Yugoslavia\n• Use clickable buttons or type country name\n• '?' if unknown",
            'relationship': "**Household Relationship (6 categories):**\n• Husband, Wife\n• Own-child\n• Not in family\n• Other relative\n• Unmarried"
        }
        
        return help_texts.get(field, f"Please provide your {field.replace('_', ' ')} information.")