Update counselor.py

counselor.py

@@ -1,18 +1,14 @@
 #!/usr/bin/env python3
 """
-UltraAdvancedHybridCounselor - Adaptive output formatting, optional lightweight
-online fine-tuning loop, and mandatory geo/language search protocol.
-
-Key behavior changes from the original:
-- Output format is chosen per-query (brief/plain, markdown_detailed, structured_xml).
-- The LLM prompt instructs different formats depending on heuristics and explicit user requests.
-- Multiple parsers accept XML-like, markdown, or plain text outputs and produce a unified structured response.
-- Machine-readable XML is produced only when explicitly requested.
-
-Caveats:
-- This keeps the fine-tune persistence and background worker.
-- Behavior is conservative: when web context or "with sources" is requested we prefer detailed markdown.
+UltraAdvancedHybridCounselor - PREMIUM Edition with Intelligent Adaptive Formatting
+
+Key improvements:
+- Intelligent query type detection (quick, definition, list, howto, comparison, etc.)
+- Adaptive output formatting based on query type
+- No forced structure - responses feel natural and premium
+- Proper markdown with line breaks preserved
 """
+
 import asyncio
 import hashlib
 import inspect
@@ -41,12 +37,14 @@ try:
     import joblib
 except Exception:
     joblib = None
+
 try:
     import torch
     from torch.utils.data import TensorDataset, DataLoader, RandomSampler
     from torch.optim import AdamW
 except Exception:
     torch = None
+
 try:
     from transformers import DistilBertTokenizer, DistilBertForSequenceClassification
 except Exception:
@@ -64,24 +62,28 @@ except Exception:
     LLMChain = None
 
 try:
-    from langchain_community.retrievers import TavilySearchAPIRetriever  # type: ignore
+    from langchain_community.retrievers import TavilySearchAPIRetriever
     _TAVILY_CLASS = TavilySearchAPIRetriever
 except Exception:
     _TAVILY_CLASS = None
+
 try:
     from rag import RAGComponent
 except Exception:
     RAGComponent = None
+
 try:
     from db import SessionDB
 except Exception:
     SessionDB = None
+
 try:
     from cache import RedisCache
 except Exception:
     RedisCache = None
+
 try:
-    from tavily import UsageLimitExceededError  # type: ignore
+    from tavily import UsageLimitExceededError
 except Exception:
     class UsageLimitExceededError(Exception):
         pass
@@ -95,35 +97,281 @@ logging.basicConfig(
     format='%(asctime)s - %(name)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s',
     handlers=[logging.FileHandler('logs/counselor.log', encoding='utf-8'), logging.StreamHandler()]
 )
+
 logger = logging.getLogger(__name__)
 
-# --- SimpleDoc and Tavily helpers ---
+
+# ============================================
+# QUERY TYPE DETECTION - THE KEY INNOVATION
+# ============================================
+
+class QueryType:
+    QUICK = "quick"              # "What is X?" - 1-2 sentence answer
+    DEFINITION = "definition"    # "Define X" / "Explain X" - short paragraph
+    LIST = "list"                # "Give me list of..." / "Top 10..." - bullet points
+    HOWTO = "howto"              # "How to..." / "Steps to..." - numbered steps
+    COMPARISON = "comparison"    # "X vs Y" / "Difference between..." - table/comparison
+    ROADMAP = "roadmap"          # "Roadmap for..." / "Path to become..." - timeline
+    SYLLABUS = "syllabus"        # "Syllabus for..." / "Curriculum..." - structured list
+    DETAILED = "detailed"        # Complex questions needing full explanation
+    CONVERSATIONAL = "conversational"  # Casual chat, follow-ups
+
+
+def detect_query_type(query: str) -> str:
+    """Intelligently detect the type of query to format response appropriately."""
+    q = query.lower().strip()
+    words = q.split()
+    word_count = len(words)
+    
+    # Quick answers (very short questions about facts)
+    quick_patterns = [
+        r"^what is [a-z\s]{1,30}\??$",
+        r"^who is ",
+        r"^when (is|was|did) ",
+        r"^where (is|was) ",
+        r"^is [a-z\s]+ (a|an) ",
+    ]
+    for pattern in quick_patterns:
+        if re.match(pattern, q) and word_count <= 6:
+            return QueryType.QUICK
+    
+    # Definition requests
+    if any(q.startswith(p) for p in ["define ", "explain ", "what does ", "meaning of "]):
+        return QueryType.DEFINITION
+    
+    # Syllabus/Curriculum requests
+    syllabus_triggers = ["syllabus", "curriculum", "course content", "course outline", "topics covered", "what to study"]
+    if any(t in q for t in syllabus_triggers):
+        return QueryType.SYLLABUS
+    
+    # Roadmap requests
+    roadmap_triggers = ["roadmap", "path to become", "how to become", "career path", "learning path", 
+                        "journey to", "steps to become", "guide to become"]
+    if any(t in q for t in roadmap_triggers):
+        return QueryType.ROADMAP
+    
+    # List requests
+    list_triggers = ["list of", "give me list", "top 10", "top 5", "best ", "names of", 
+                     "examples of", "types of", "kinds of", "options for"]
+    if any(t in q for t in list_triggers):
+        return QueryType.LIST
+    
+    # How-to requests
+    howto_triggers = ["how to ", "how do i ", "how can i ", "steps to ", "process of ", 
+                      "guide for ", "tutorial", "way to "]
+    if any(t in q for t in howto_triggers):
+        return QueryType.HOWTO
+    
+    # Comparison requests
+    if " vs " in q or " versus " in q or "difference between" in q or "compare " in q:
+        return QueryType.COMPARISON
+    
+    # Conversational (short casual queries)
+    if word_count <= 4:
+        return QueryType.CONVERSATIONAL
+    
+    # Default to detailed for complex questions
+    return QueryType.DETAILED
+
+
+# ============================================
+# ADAPTIVE PROMPT TEMPLATES
+# ============================================
+
+def get_adaptive_prompt(query_type: str, persona: str, intent: str, web_context: str, 
+                        rag_context: str, history_str: str, user_query: str) -> str:
+    """Generate query-type-specific prompts that produce natural responses."""
+    
+    tone_instruction = {
+        "Student": "Use a friendly, encouraging tone with practical examples.",
+        "Teacher": "Use a professional, resourceful tone with academic references.",
+        "Parent": "Use an empathetic, clear tone with actionable guidance.",
+        "Other": "Use a helpful, informative tone."
+    }.get(persona, "Use a helpful, informative tone.")
+    
+    # Base context section
+    context_section = ""
+    if web_context and web_context != "No web search required or performed.":
+        context_section = f"""
+**Available Context:**
+{web_context}
+"""
+    
+    # Query-type specific instructions
+    if query_type == QueryType.QUICK:
+        format_instruction = """
+**Response Format:** Give a direct, concise answer in 1-2 sentences. No headers, no bullet points, no lengthy explanations. Just answer the question naturally like a knowledgeable friend would.
+
+Example: "Machine learning is a type of artificial intelligence that enables computers to learn from data and improve their performance without being explicitly programmed."
+"""
+    
+    elif query_type == QueryType.DEFINITION:
+        format_instruction = """
+**Response Format:** Provide a clear definition in 2-3 sentences, followed by a brief practical example or application. Keep it conversational. No headers needed.
+
+Example format:
+[Definition in 2-3 sentences]
+
+[One practical example or real-world application]
+"""
+    
+    elif query_type == QueryType.SYLLABUS:
+        format_instruction = """
+**Response Format:** Present the syllabus as a clean, numbered list. Each topic should be on its own line. Group related topics under clear section headers if needed.
+
+**IMPORTANT:** Each numbered item MUST be on a NEW LINE. Use this exact format:
+
+**[Subject Name] Syllabus:**
+
+**1. [First Topic/Module]**
+   - Subtopic A
+   - Subtopic B
+
+**2. [Second Topic/Module]**
+   - Subtopic A
+   - Subtopic B
+
+Continue this pattern. Keep descriptions brief (1 line each).
+"""
+    
+    elif query_type == QueryType.ROADMAP:
+        format_instruction = """
+**Response Format:** Present as a clear timeline/path. Use phases or stages with timeframes where appropriate.
+
+**Format like this:**
+
+### Phase 1: Foundation (Month 1-2)
+- Skill/topic to learn
+- Skill/topic to learn
+
+### Phase 2: Intermediate (Month 3-4)
+- Skill/topic to learn
+- Skill/topic to learn
+
+### Phase 3: Advanced (Month 5-6)
+- Skill/topic to learn
+- Skill/topic to learn
+
+**Resources:** List 2-3 recommended resources at the end.
+"""
+    
+    elif query_type == QueryType.LIST:
+        format_instruction = """
+**Response Format:** Present as a clean bullet list. Each item on its own line.
+
+**Format:**
+Here are the [requested items]:
+
+- **Item 1:** Brief description
+- **Item 2:** Brief description
+- **Item 3:** Brief description
+
+Keep descriptions to 1 line each. No lengthy paragraphs.
+"""
+    
+    elif query_type == QueryType.HOWTO:
+        format_instruction = """
+**Response Format:** Present as numbered steps. Each step should be actionable and clear.
+
+**Format:**
+Here's how to [do the thing]:
+
+**Step 1: [Action verb + what to do]**
+Brief explanation (1-2 sentences max).
+
+**Step 2: [Action verb + what to do]**
+Brief explanation (1-2 sentences max).
+
+Continue this pattern. Keep it practical and actionable.
+"""
+    
+    elif query_type == QueryType.COMPARISON:
+        format_instruction = """
+**Response Format:** Present a clear comparison. You can use a simple table format or side-by-side comparison.
+
+**Format:**
+Here's how [X] and [Y] compare:
+
+| Aspect | X | Y |
+|--------|---|---|
+| [Aspect 1] | [X's characteristic] | [Y's characteristic] |
+| [Aspect 2] | [X's characteristic] | [Y's characteristic] |
+
+**Bottom Line:** 1-2 sentences on when to choose each option.
+"""
+    
+    elif query_type == QueryType.CONVERSATIONAL:
+        format_instruction = """
+**Response Format:** Keep it brief and conversational. 1-3 sentences max. No headers, no bullet points unless specifically asked. Respond like a helpful friend would in a chat.
+"""
+    
+    else:  # DETAILED
+        format_instruction = """
+**Response Format:** Provide a comprehensive answer with clear structure:
+
+1. Start with a direct 2-3 sentence answer to the main question
+2. Use headers (##) only if covering multiple distinct aspects
+3. Use bullet points for lists
+4. Keep paragraphs short (3-4 sentences max)
+5. End with practical advice or next steps if relevant
+
+Keep the response focused and avoid unnecessary filler.
+"""
+    
+    prompt = f"""You are a premium AI career and education counselor. {tone_instruction}
+
+**User's Question:** {user_query}
+
+{context_section}
+
+**Conversation History:**
+{history_str}
+
+{format_instruction}
+
+**Critical Rules:**
+1. NEVER use XML tags like <summary>, <explanation>, <insights>
+2. NEVER force a rigid structure if it doesn't fit the question
+3. Preserve proper line breaks - each list item/step MUST be on its own line
+4. Match your response length to the complexity of the question
+5. Be helpful, accurate, and natural
+
+Now respond to the user's question:"""
+    
+    return prompt
+
+
+# ============================================
+# SIMPLE DOC AND TAVILY HELPERS (Unchanged)
+# ============================================
+
 class SimpleDoc:
     def __init__(self, source: str, content: str, title: str = "", score: float = None):
         self.metadata = {"source": source, "title": title, "score": score}
         self.page_content = content
 
+
 def create_tavily_retriever_safe(k: int = 10, logger: logging.Logger = logger, **kwargs):
     global _TAVILY_CLASS
     if _TAVILY_CLASS is None:
         try:
-            from langchain_community.retrievers import TavilySearchAPIRetriever  # type: ignore
+            from langchain_community.retrievers import TavilySearchAPIRetriever
             _TAVILY_CLASS = TavilySearchAPIRetriever
         except Exception as e:
             logger.error(f"TavilySearchAPIRetriever not importable: {e}")
             raise ImportError("TavilySearchAPIRetriever unavailable") from e
-
+    
     cls = _TAVILY_CLASS
     try:
         sig = inspect.signature(cls.__init__)
     except Exception:
         sig = None
-
+    
     allowed = {}
     for name, val in {"k": k, **kwargs}.items():
         if sig is None or (name in sig.parameters and name != "self"):
             allowed[name] = val
-
+    
     try:
         return cls(**allowed)
     except TypeError as te:
@@ -134,14 +382,15 @@ def create_tavily_retriever_safe(k: int = 10, logger: logging.Logger = logger, *
             logger.error(f"Tavily no-arg constructor failed: {e}")
             raise
 
+
 async def tavily_search_safe(retriever, query: str, logger: logging.Logger = logger, *args, **kwargs) -> List[Any]:
     if retriever is None:
         logger.debug("tavily_search_safe: retriever is None")
         return []
-
+    
     async_methods = ["ainvoke", "aget_relevant_documents", "aretrieve", "asearch"]
     sync_methods = ["invoke", "get_relevant_documents", "retrieve", "search"]
-
+    
     for name in async_methods:
         fn = getattr(retriever, name, None)
         if callable(fn):
@@ -154,7 +403,7 @@ async def tavily_search_safe(retriever, query: str, logger: logging.Logger = log
                     continue
             except Exception:
                 continue
-
+    
     loop = asyncio.get_event_loop()
     for name in sync_methods:
         fn = getattr(retriever, name, None)
@@ -163,24 +412,26 @@ async def tavily_search_safe(retriever, query: str, logger: logging.Logger = log
                 return await loop.run_in_executor(None, lambda: fn(query))
             except Exception:
                 continue
-
+    
     logger.warning("tavily_search_safe: no usable methods on retriever")
     return []
 
+
 async def tavily_rest_search(api_key: str, query: str, timeout: int = 15, logger: logging.Logger = logger) -> List[Dict[str, Any]]:
     if requests is None:
         logger.error("requests library not available; cannot use REST fallback for Tavily.")
         return []
+    
     url = "https://api.tavily.com/search"
     headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
     payload = {"query": query}
     loop = asyncio.get_event_loop()
-
+    
     def do_post():
         r = requests.post(url, json=payload, headers=headers, timeout=timeout)
         r.raise_for_status()
         return r.json()
-
+    
     try:
         resp = await loop.run_in_executor(None, do_post)
         results = resp.get("results", [])
@@ -190,115 +441,51 @@ async def tavily_rest_search(api_key: str, query: str, timeout: int = 15, logger
         logger.exception("tavily_rest_search failed")
         return []
 
+
 def format_sources_block(docs: List[SimpleDoc]) -> str:
     if not docs:
         return ""
-    lines = ["\n---\n### 📄 Sources used for this answer:"]
-    for d in docs:
+    lines = ["\n---\n**📚 Sources:**"]
+    for i, d in enumerate(docs[:5], 1):  # Limit to 5 sources
         meta = getattr(d, "metadata", {}) or {}
         url = meta.get("source") or ""
-        title = meta.get("title") or ""
-        score = meta.get("score")
-        if score is None:
-            score_display = ""
-        else:
-            try:
-                score_display = f"⭐ Relevance: {float(score):.3f}"
-            except Exception:
-                score_display = f"⭐ Relevance: {score}"
-        lines.append(f"🔗 **URL:** {url}")
-        if title:
-            lines.append(f"   **Title:** {title}")
-        if score_display:
-            lines.append(f"   **Relevance:** {score_display}")
-        lines.append("")
-    return "\n".join(lines).strip()
-
-def _parse_llm_tagged_output(text: str) -> Dict[str, str]:
-    data = {}
-    tags = {
-        r"<SUMMARY>(.*?)</SUMMARY>": "summary",
-        r"<COMPREHENSIVE_EXPLANATION>(.*?)</COMPREHENSIVE_EXPLANATION>": "explanation",
-        r"<RELEVANT_INSIGHTS>(.*?)</RELEVANT_INSIGHTS>": "insights",
-    }
-    for pattern, key in tags.items():
-        match = re.search(pattern, text, re.DOTALL | re.IGNORECASE)
-        if match:
-            content = match.group(1).strip()
-            if content:
-                data[key] = content
-    if "explanation" not in data or not data["explanation"]:
-        clean_text = re.sub(r"<[^>]+>", "", text).strip()
-        if "summary" in data and data["summary"] in clean_text:
-            clean_text = clean_text.replace(data["summary"], "").strip()
-        data["explanation"] = clean_text or "No detailed answer was generated."
-    return data
-
-# --- New: parsing helpers to support multiple output formats ---
-def _parse_markdown_structured(text: str) -> Dict[str, str]:
-    # Look for headings like "Summary", "Detailed Explanation", "Relevant Insights"
-    data = {"summary": "", "explanation": "", "insights": ""}
-    t = text.replace("\r\n", "\n")
-    summary_match = re.search(r"(^|\n)#+\s*Summary\s*\n(.*?)(\n#|\n\Z)", t, re.IGNORECASE | re.DOTALL)
-    if summary_match:
-        data["summary"] = summary_match.group(2).strip()
-    else:
-        m = re.search(r"(^|\n)Summary\s*[:\-]\s*(.*?)(\n\n|\n#|\n\Z)", t, re.IGNORECASE | re.DOTALL)
-        if m:
-            data["summary"] = m.group(2).strip()
-
-    expl_match = re.search(r"(^|\n)#+\s*(Detailed Explanation|Explanation|Answer)\s*\n(.*?)(\n#|\n\Z)", t, re.IGNORECASE | re.DOTALL)
-    if expl_match:
-        data["explanation"] = expl_match.group(2).strip()
-    else:
-        if data["summary"]:
-            parts = t.split(data["summary"], 1)
-            if len(parts) > 1:
-                data["explanation"] = parts[1].strip()
-        else:
-            data["explanation"] = t.strip()
-
-    insights_match = re.search(r"(^|\n)#+\s*Relevant Insights\s*\n(.*?)(\n#|\n\Z)", t, re.IGNORECASE | re.DOTALL)
-    if insights_match:
-        data["insights"] = insights_match.group(2).strip()
-    else:
-        m2 = re.search(r"(^|\n)Relevant Insights\s*[:\-]\s*(.*?)(\n#|\n\Z)", t, re.IGNORECASE | re.DOTALL)
-        if m2:
-            data["insights"] = m2.group(2).strip()
-    for k in data:
-        if not data[k]:
-            data[k] = ""
-    return data
-
-def _parse_plain_text(text: str) -> Dict[str, str]:
-    s = text.strip()
-    if not s:
-        return {"summary": "", "explanation": "", "insights": ""}
-    sentences = re.split(r'(?<=[.!?])\s+', s)
-    summary = " ".join(sentences[:2]).strip()
-    explanation = " ".join(sentences[2:]).strip() if len(sentences) > 2 else ""
-    if not explanation:
-        explanation = summary
-    return {"summary": summary, "explanation": explanation, "insights": ""}
-
-# --- Keyword Lists ---
+        title = meta.get("title") or url
+        if url:
+            lines.append(f"{i}. [{title}]({url})")
+    return "\n".join(lines)
+
+
+# ============================================
+# KEYWORD LISTS (Unchanged)
+# ============================================
+
 _COUNTRY_KEYWORDS = {
-    "india", "usa", "united states", "canada", "uk", "united kingdom", "germany", "france", "japan", "china", "brazil",
-    "australia", "singapore", "netherlands", "italy", "spain"
+    "india", "usa", "united states", "canada", "uk", "united kingdom", "germany", 
+    "france", "japan", "china", "brazil", "australia", "singapore", "netherlands", 
+    "italy", "spain"
 }
+
 _LANGUAGE_KEYWORDS = {
-    "english", "german", "french", "spanish", "mandarin", "chinese", "japanese", "korean", "hindi", "arabic", "portuguese"
+    "english", "german", "french", "spanish", "mandarin", "chinese", "japanese", 
+    "korean", "hindi", "arabic", "portuguese"
 }
+
 _ILLEGAL_TRIGGERS = [
-    r"how to make a bomb", r"detonate", r"how to assassinat", r"kill someone", r"poison", r"how to hack into",
-    r"bypass security", r"carding", r"credit card fraud", r"explosive", r"illicit drug", r"how to sell drugs",
-    r"manufacture illegal", r"produce illegal", r"evade law", r"how to avoid taxes illegally"
+    r"how to make a bomb", r"detonate", r"how to assassinat", r"kill someone", 
+    r"poison", r"how to hack into", r"bypass security", r"carding", r"credit card fraud", 
+    r"explosive", r"illicit drug", r"how to sell drugs", r"manufacture illegal", 
+    r"produce illegal", r"evade law", r"how to avoid taxes illegally"
 ]
 
+
+# ============================================
+# MAIN COUNSELOR CLASS
+# ============================================
+
 class UltraAdvancedHybridCounselor:
     def __init__(self):
         logger.info(f"🐍 Python version: {sys.version}")
-
+        
         # --- Paths and Model State ---
         self.model_path = "Sachin21112004/carrerflow-ai"
         self.label_encoder_path = "Sachin21112004/carrerflow-ai/label_encoder.pkl"
@@ -316,30 +503,67 @@ class UltraAdvancedHybridCounselor:
             "gemini-1.5-pro-002", "gemini-2.5-flash-lite-preview", "gemini-1.5-flash-8b-latest",
             "gemini-1.5-flash-latest", "gemini-1.5-pro-latest", "gemini-1.0-pro", "gemini-pro"
         ]
-
+        
         # --- Persistent user-adaptation files ---
         self.user_corpus_path = Path("user_corpus.txt")
         self.user_keywords_path = Path("user_keywords.json")
         self.user_greetings_path = Path("user_greetings.json")
-
+        
         # Fine-tuning dataset and config
         self.finetune_examples_path = Path("fine_tune_examples.jsonl")
         self.finetune_label_map_path = Path("fine_tune_label_map.json")
-
-        # --- New HF Dataset Config ---
+        
+        # --- HF Dataset Config ---
         self.dataset_repo_id = os.getenv("HF_DATASET_REPO_ID", "Sachin21112004/DreamFlow-AI-Data")
         self.examples_filename_in_repo = "fine_tune_examples.jsonl"
-        self.local_examples_path = Path(f"./{self.examples_filename_in_repo}")  # Temp local file
-
+        self.local_examples_path = Path(f"./{self.examples_filename_in_repo}")
         self.fine_tune_interval = int(os.getenv("FINE_TUNE_INTERVAL_SECS", "300"))
         self.min_examples_to_train = int(os.getenv("MIN_EXAMPLES_TO_TRAIN", "32"))
         self.fine_tune_batch_size = int(os.getenv("FINE_TUNE_BATCH", "8"))
         self.fine_tune_epochs = int(os.getenv("FINE_TUNE_EPOCHS", "1"))
-
+        
         # Default greetings
-        self._default_greetings = {"hi", "hiii", "hii", "hello", "hey", "hlo", "how are you", "good morning", "good afternoon", "good evening"}
-
+        self._default_greetings = {
+            "hi", "hiii", "hii", "hello", "hey", "hlo", "how are you", 
+            "good morning", "good afternoon", "good evening"
+        }
+        
         # Load persisted greetings and user keywords
+        self._load_user_data()
+        
+        # --- Load local ML model (if available) ---
+        self._load_local_models()
+        
+        # --- Initialize RAG and DB ---
+        self._initialize_rag_db()
+        
+        # --- Tavily key rotation setup ---
+        self._initialize_tavily()
+        
+        # --- Redis caching ---
+        self._initialize_cache()
+        
+        # --- Initialize LLM ---
+        try:
+            self.llm = self._initialize_llm()
+            if self.llm:
+                logger.info(f"✅ LLM initialized: {self.current_model}")
+            else:
+                logger.info("LLM not initialized; operating in degraded mode.")
+        except Exception as e:
+            logger.error(f"LLM initialization error: {e}")
+            self.llm = None
+        
+        # Setup intent chain
+        self._setup_intent_chain()
+        
+        # --- Start background fine-tune worker ---
+        self._start_fine_tune_worker()
+        
+        logger.info("✅ UltraAdvancedHybridCounselor PREMIUM Edition ready.")
+
+    def _load_user_data(self):
+        """Load persisted user greetings and keywords."""
         try:
             if self.user_greetings_path.exists():
                 with open(self.user_greetings_path, "r", encoding="utf-8") as f:
@@ -347,7 +571,7 @@ class UltraAdvancedHybridCounselor:
                     self.greetings = set(stored.get("greetings", [])) | self._default_greetings
             else:
                 self.greetings = set(self._default_greetings)
-
+            
             if self.user_keywords_path.exists():
                 with open(self.user_keywords_path, "r", encoding="utf-8") as f:
                     self.user_keywords = json.load(f)
@@ -358,7 +582,8 @@ class UltraAdvancedHybridCounselor:
             self.greetings = set(self._default_greetings)
             self.user_keywords = {}
 
-        # --- Load local ML model (if available) ---
+    def _load_local_models(self):
+        """Load local ML models for intent classification."""
         try:
             if DistilBertTokenizer and DistilBertForSequenceClassification:
                 try:
@@ -370,29 +595,35 @@ class UltraAdvancedHybridCounselor:
                         self.tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased")
                     except Exception:
                         self.tokenizer = None
-                        logger.debug("No tokenizer available locally.")
+                    logger.debug("No tokenizer available locally.")
+            
             if joblib and Path(self.label_encoder_path).exists():
                 self.label_encoder = joblib.load(self.label_encoder_path)
                 logger.info("✅ Label encoder loaded")
         except Exception as e:
             logger.error(f"Error loading local ML models: {e}")
 
-        # --- Initialize RAG and DB ---
+    def _initialize_rag_db(self):
+        """Initialize RAG and SessionDB components."""
         try:
             self.rag = RAGComponent() if RAGComponent else None
             self.db = SessionDB() if SessionDB else None
-            if self.rag: logger.info("✅ RAG initialized")
-            if self.db: logger.info("✅ SessionDB initialized")
+            if self.rag:
+                logger.info("✅ RAG initialized")
+            if self.db:
+                logger.info("✅ SessionDB initialized")
         except Exception as e:
             logger.error(f"Error initializing RAG/DB: {e}")
             self.rag = None
             self.db = None
 
-        # --- Tavily key rotation setup ---
+    def _initialize_tavily(self):
+        """Initialize Tavily search with key rotation."""
         self.tavily = None
         self.tavily_keys_list = []
         self.tavily_key_pool = None
         self.current_tavily_key = None
+        
         try:
             tavily_keys_str = os.getenv("TAVILY_API_KEY", "")
             if tavily_keys_str:
@@ -407,7 +638,8 @@ class UltraAdvancedHybridCounselor:
             logger.error(f"Error during Tavily init: {e}")
             self.tavily = None
 
-        # --- Redis caching ---
+    def _initialize_cache(self):
+        """Initialize Redis cache if available."""
         self.use_redis = os.getenv("USE_REDIS", "false").lower() == "true"
         self.cache = None
         if self.use_redis and RedisCache:
@@ -416,34 +648,8 @@ class UltraAdvancedHybridCounselor:
             except Exception:
                 pass
 
-        # --- Initialize LLM ---
-        try:
-            self.llm = self._initialize_llm()
-            if self.llm: logger.info(f"✅ LLM initialized: {self.current_model}")
-            else: logger.info("LLM not initialized; operating in degraded mode.")
-        except Exception as e:
-            logger.error(f"LLM initialization error: {e}")
-            self.llm = None
-
-        # Setup prompts/chains
-        self._setup_prompts()
-
-        # --- Start background fine-tune worker if local training is possible ---
-        self._fine_tune_lock = threading.Lock()
-        self._stop_fine_tune_worker = False
-        self._fine_tune_thread = None
-        if torch and self.model is not None and self.tokenizer is not None:
-            try:
-                self._fine_tune_thread = threading.Thread(target=self._fine_tune_loop_sync, daemon=True)
-                self._fine_tune_thread.start()
-                logger.info("✅ Background fine-tune worker started.")
-            except Exception as e:
-                logger.error(f"Failed to start fine-tune background worker: {e}")
-
-        logger.info("UltraAdvancedHybridCounselor ready.")
-
-    # --- LLM and Tavily Utility Methods ---
     def _get_model_priority_score(self, model_name: str) -> int:
+        """Get priority score for model selection."""
         priority_map = {
             "gemini-2.5-flash-lite": 100, "gemini-2.5-flash": 95, "gemini-2.0-flash-lite": 90,
             "gemini-2.0-flash": 85, "gemini-2.5-pro": 80, "gemini-1.5-flash": 75, "gemini-1.5-pro": 60
@@ -451,44 +657,60 @@ class UltraAdvancedHybridCounselor:
         return priority_map.get(model_name, 10)
 
     def _initialize_llm(self):
+        """Initialize LLM with fallback support."""
         google_api_key = os.getenv("GOOGLE_API_KEY")
         if not google_api_key or ChatGoogleGenerativeAI is None:
             return None
-
+        
         sorted_models = sorted(self.available_models, key=self._get_model_priority_score, reverse=True)
         for model_name in sorted_models:
             try:
                 llm = ChatGoogleGenerativeAI(
-                    model=model_name, temperature=0.1, max_tokens=4096,
+                    model=model_name, temperature=0.3, max_tokens=4096,
                     google_api_key=google_api_key, timeout=30, max_retries=1
                 )
                 if hasattr(llm, "invoke"):
                     _ = llm.invoke("ping")
                 elif hasattr(llm, "generate"):
                     _ = llm.generate("ping")
+                
                 self.current_model = model_name
-                self.model_performance_stats[model_name] = {"response_time": 0.0, "success_rate": 1.0, "last_used": time.time(), "total_requests": 0, "successful_requests": 0}
+                self.model_performance_stats[model_name] = {
+                    "response_time": 0.0, "success_rate": 1.0, 
+                    "last_used": time.time(), "total_requests": 0, "successful_requests": 0
+                }
                 return llm
             except Exception:
                 continue
+        
         logger.error("No LLM models could be initialized.")
         return None
 
     def _fallback_to_next_model(self) -> bool:
+        """Attempt to fallback to next available model."""
         if ChatGoogleGenerativeAI is None:
             return False
+        
         try:
             current_index = self.available_models.index(self.current_model) if self.current_model in self.available_models else -1
             remaining = self.available_models[current_index + 1:] if current_index >= 0 else self.available_models
         except Exception:
             remaining = self.available_models
+        
         remaining = sorted(remaining, key=self._get_model_priority_score, reverse=True)
         for model in remaining:
             try:
-                llm = ChatGoogleGenerativeAI(model=model, temperature=0.1, max_tokens=4096, google_api_key=os.getenv("GOOGLE_API_KEY"), timeout=30, max_retries=1)
-                if hasattr(llm, "invoke"): _ = llm.invoke("ping")
-                elif hasattr(llm, "generate"): _ = llm.generate("ping")
-                else: continue
+                llm = ChatGoogleGenerativeAI(
+                    model=model, temperature=0.3, max_tokens=4096,
+                    google_api_key=os.getenv("GOOGLE_API_KEY"), timeout=30, max_retries=1
+                )
+                if hasattr(llm, "invoke"):
+                    _ = llm.invoke("ping")
+                elif hasattr(llm, "generate"):
+                    _ = llm.generate("ping")
+                else:
+                    continue
+                
                 self.llm = llm
                 self.current_model = model
                 logger.info(f"Fell back to {model}")
@@ -498,50 +720,156 @@ class UltraAdvancedHybridCounselor:
         return False
 
     def _update_model_stats(self, model_name: str, success: bool, response_time: float = None, error: str = None):
+        """Update model performance statistics."""
         if model_name not in self.model_performance_stats:
-            self.model_performance_stats[model_name] = {"total_requests": 0, "successful_requests": 0, "response_time": None, "success_rate": 0.0}
+            self.model_performance_stats[model_name] = {
+                "total_requests": 0, "successful_requests": 0, 
+                "response_time": None, "success_rate": 0.0
+            }
+        
         stats = self.model_performance_stats[model_name]
         stats["total_requests"] = stats.get("total_requests", 0) + 1
+        
         if success:
             stats["successful_requests"] = stats.get("successful_requests", 0) + 1
             stats["response_time"] = response_time
             stats["last_used"] = time.time()
         else:
-            if error: stats["last_error"] = error
+            if error:
+                stats["last_error"] = error
+        
         total = stats["total_requests"]
         stats["success_rate"] = stats.get("successful_requests", 0) / total if total > 0 else 0.0
 
-    async def _safe_llm_invoke(self, chain, params: Dict[str, Any], max_retries: int = 2) -> Any:
-        if chain is None or self.llm is None:
-            raise RuntimeError("LLM chain or LLM not available.")
-        for attempt in range(max_retries):
+    def _setup_intent_chain(self):
+        """Setup intent classification chain."""
+        self.intent_chain = None
+        if self.llm and LLMChain and PromptTemplate:
+            self.intent_template = """You are an intent classifier. Respond only with one of: educational_guidance, career_recommendation, resume_advice, interview_prep, salary_info, general_guidance, off_topic. Query: {query}"""
             try:
-                start = time.time()
-                if hasattr(chain, "ainvoke"):
-                    res = await chain.ainvoke(params)
-                else:
-                    loop = asyncio.get_event_loop()
-                    res = await loop.run_in_executor(None, lambda: chain.invoke(params))
-                self._update_model_stats(self.current_model, True, time.time() - start)
-                return res
+                self.intent_chain = LLMChain(llm=self.llm, prompt=PromptTemplate.from_template(self.intent_template))
             except Exception as e:
-                self._update_model_stats(self.current_model, False, error=str(e))
-                msg = str(e).lower()
-                if any(k in msg for k in ["not found", "404", "not supported", "invalid model", "model does not exist"]):
-                    if self._fallback_to_next_model():
-                        logger.info("Retrying after fallback model selection.")
-                        continue
-                    else:
-                        raise RuntimeError("All models failed.")
-                if attempt < max_retries - 1:
-                    await asyncio.sleep(2 ** attempt)
-                else:
-                    raise
+                logger.error(f"Failed to create intent chain: {e}")
+                self.intent_chain = None
+
+    def _start_fine_tune_worker(self):
+        """Start background fine-tuning worker thread."""
+        self._fine_tune_lock = threading.Lock()
+        self._stop_fine_tune_worker = False
+        self._fine_tune_thread = None
+        
+        if torch and self.model is not None and self.tokenizer is not None:
+            try:
+                self._fine_tune_thread = threading.Thread(target=self._fine_tune_loop_sync, daemon=True)
+                self._fine_tune_thread.start()
+                logger.info("✅ Background fine-tune worker started.")
+            except Exception as e:
+                logger.error(f"Failed to start fine-tune background worker: {e}")
+
+    # ============================================
+    # CORE HEURISTICS
+    # ============================================
+
+    def _should_use_web_search(self, intent: str, query: str, history: List[Any] = None) -> bool:
+        """Determine if web search should be used for this query."""
+        if not query:
+            return False
+        
+        q = query.lower()
+        
+        # Force triggers
+        force_triggers = [
+            "with sources", "with source", "show sources", "cite", "sources", 
+            "verify", "search web", "web search", "please search", "please look up", 
+            "look up", "confirm from", "confirm that"
+        ]
+        if any(t in q for t in force_triggers):
+            return True
+        
+        if intent == "salary_info":
+            return True
+        
+        # Web triggers
+        web_triggers = [
+            "latest", "current", "202", "trend", "trends", "salary", "average", 
+            "median", "top", "emerging", "statistics", "how much", "pay", "ctc", 
+            "package", "percent", "percentile", "growth", "outlook"
+        ]
+        if any(w in q for w in web_triggers):
+            return True
+        
+        # Skip web search for greetings and short queries
+        if q.strip() in self.greetings or len(q.split()) <= 4:
+            return False
+        
+        return False
+
+    def _is_illegal_request(self, query: str) -> bool:
+        """Check if the request is for illegal content."""
+        if not query:
+            return False
+        
+        q = query.lower()
+        for pattern in _ILLEGAL_TRIGGERS:
+            if re.search(pattern, q):
+                return True
+        
+        risky = ["how to make", "how to build a", "how to bypass", "how to hack", "evade", "explosive", "make poison", "sell drugs"]
+        if any(r in q for r in risky) and any(word in q for word in ["bomb", "poison", "explode", "assassin", "hack", "illicit", "illegal", "fraud"]):
+            return True
+        
+        return False
+
+    def _detect_country_language(self, query: str) -> Dict[str, Optional[str]]:
+        """Detect country and language mentions in query."""
+        if not query:
+            return {"country": None, "language": None}
+        
+        q = query.lower()
+        country_found = next((c for c in _COUNTRY_KEYWORDS if re.search(r"\b" + re.escape(c) + r"\b", q)), None)
+        language_found = next((l for l in _LANGUAGE_KEYWORDS if re.search(r"\b" + re.escape(l) + r"\b", q)), None)
+        return {"country": country_found, "language": language_found}
+
+    def _detect_persona(self, query: str) -> str:
+        """Detect user persona from query."""
+        if not query:
+            return "Other"
+        
+        q = query.lower()
+        if any(k in q for k in ["i am a student", "student", "grade", "class", "college", "undergraduate", "btech", "mba", "high school"]):
+            return "Student"
+        if any(k in q for k in ["i am a teacher", "teacher", "instructor", "professor", "lecturer"]):
+            return "Teacher"
+        if any(k in q for k in ["my child", "parent", "mother", "father", "guardian"]):
+            return "Parent"
+        return "Other"
+
+    def _generate_mandatory_search_queries(self, user_query: str, country: Optional[str], language: Optional[str]) -> List[str]:
+        """Generate search queries based on geo/language context."""
+        searches = []
+        base = user_query.strip()
+        
+        if country:
+            searches.append(f"{base} {country} official requirements site:gov OR site:.edu")
+            searches.append(f"{base} {country} curriculum requirements OR regulations")
+        elif language:
+            searches.append(f"{base} {language} language learning resources official exams")
+            searches.append(f"{base} {language} proficiency exam requirements OR recognized certifications")
+        else:
+            searches.append(f"{base} official guidance")
+            searches.append(f"{base} statistics OR latest data")
+        
+        return list(set(searches))
+
+    # ============================================
+    # LLM INVOCATION
+    # ============================================
 
     async def _call_direct_llm(self, prompt: str, max_retries: int = 2) -> str:
+        """Call LLM directly with retry logic."""
         if self.llm is None:
             return "LLM not available. Enable GOOGLE_API_KEY and ensure dependencies are installed."
-
+        
         for attempt in range(max_retries):
             try:
                 start = time.time()
@@ -554,10 +882,10 @@ class UltraAdvancedHybridCounselor:
                     full_response_text = res.content if hasattr(res, 'content') else str(res)
                 else:
                     return "LLM present but has no recognized call method."
-
+                
                 self._update_model_stats(self.current_model, True, time.time() - start)
                 return full_response_text
-
+            
             except Exception as e:
                 self._update_model_stats(self.current_model, False, error=str(e))
                 msg = str(e).lower()
@@ -567,27 +895,36 @@ class UltraAdvancedHybridCounselor:
                         continue
                     else:
                         raise RuntimeError("All models failed.")
+                
                 if attempt < max_retries - 1:
                     await asyncio.sleep(2 ** attempt)
                 else:
                     logger.error(f"Direct LLM call failed after {max_retries} attempts: {e}")
                     raise
+        
         return "I encountered an error while generating the response after multiple retries."
 
+    # ============================================
+    # TAVILY SEARCH WITH ROTATION
+    # ============================================
+
     async def _rotate_tavily_key(self, query: str, max_retries: int = None) -> list:
+        """Perform Tavily search with key rotation on failure."""
         if not getattr(self, "tavily_key_pool", None) or not getattr(self, "tavily_keys_list", None):
             return []
+        
         if max_retries is None:
             max_retries = min(3, len(self.tavily_keys_list))
+        
         for attempt in range(max_retries):
             try:
                 if self.current_tavily_key:
                     os.environ["TAVILY_API_KEY"] = self.current_tavily_key
-                    try:
-                        self.tavily = create_tavily_retriever_safe(k=10, logger=logger)
-                    except Exception:
-                        pass
-
+                try:
+                    self.tavily = create_tavily_retriever_safe(k=10, logger=logger)
+                except Exception:
+                    pass
+                
                 search_docs = await tavily_search_safe(self.tavily, query, logger=logger)
                 if search_docs:
                     normalized = []
@@ -599,13 +936,22 @@ class UltraAdvancedHybridCounselor:
                         content = getattr(doc, "page_content", None) or (doc.get("content") if isinstance(doc, dict) else str(doc))
                         normalized.append(SimpleDoc(source or "", content or "", title=title or "", score=score))
                     return normalized
-
+                
+                # Try REST fallback
                 if self.current_tavily_key:
                     rest_results = await tavily_rest_search(self.current_tavily_key, query)
                     if rest_results:
-                        normalized = [SimpleDoc(r.get("url") or r.get("source") or "", r.get("content") or r.get("title") or r.get("snippet") or str(r), title=r.get("title") or "", score=r.get("score")) for r in rest_results]
+                        normalized = [
+                            SimpleDoc(
+                                r.get("url") or r.get("source") or "",
+                                r.get("content") or r.get("title") or r.get("snippet") or str(r),
+                                title=r.get("title") or "",
+                                score=r.get("score")
+                            ) for r in rest_results
+                        ]
                         return normalized
-
+                
+                # Rotate key
                 if attempt < max_retries - 1:
                     try:
                         self.current_tavily_key = next(self.tavily_key_pool)
@@ -614,6 +960,7 @@ class UltraAdvancedHybridCounselor:
                     continue
                 else:
                     break
+            
             except UsageLimitExceededError:
                 if attempt < max_retries - 1:
                     try:
@@ -633,243 +980,371 @@ class UltraAdvancedHybridCounselor:
                     continue
                 else:
                     break
+        
         logger.error("🚫 All Tavily attempts failed. Falling back to no web context.")
         return []
 
-    # --- Core Heuristics ---
-    def _should_use_web_search(self, intent: str, query: str, history: List[Any] = None) -> bool:
-        if not query:
-            return False
-        q = query.lower()
-        force_triggers = ["with sources", "with source", "show sources", "cite", "sources", "verify", "search web", "web search", "please search", "please look up", "look up", "confirm from", "confirm that"]
-        if any(t in q for t in force_triggers):
-            return True
-        if intent == "salary_info":
-            return True
-        web_triggers = ["latest", "current", "202", "trend", "trends", "salary", "average", "median", "top", "emerging", "statistics", "how much", "pay", "ctc", "package", "percent", "percentile", "growth", "outlook"]
-        if any(w in q for w in web_triggers):
-            return True
-        greetings = self.greetings
-        if q.strip() in greetings or len(q.split()) <= 4:
-            return False
-        if intent == "career_recommendation":
-            exploratory = ["explore", "help me explore", "recommend", "what should i study", "what should i do", "interests", "skills", "i like", "i enjoy", "prefer"]
-            if any(kw in q for kw in exploratory):
-                return False
-        return False
+    # ============================================
+    # INTENT CLASSIFICATION
+    # ============================================
 
-    def _is_illegal_request(self, query: str) -> bool:
-        if not query:
-            return False
-        q = query.lower()
-        for pattern in _ILLEGAL_TRIGGERS:
-            if re.search(pattern, q):
-                return True
-        risky = ["how to make", "how to build a", "how to bypass", "how to hack", "evade", "explosive", "make poison", "sell drugs"]
-        if any(r in q for r in risky) and any(word in q for word in ["bomb", "poison", "explode", "assassin", "hack", "illicit", "illegal", "fraud"]):
-            return True
-        return False
-
-    def _detect_country_language(self, query: str) -> Dict[str, Optional[str]]:
-        if not query:
-            return {"country": None, "language": None}
-        q = query.lower()
-        country_found = next((c for c in _COUNTRY_KEYWORDS if re.search(r"\b" + re.escape(c) + r"\b", q)), None)
-        language_found = next((l for l in _LANGUAGE_KEYWORDS if re.search(r"\b" + re.escape(l) + r"\b", q)), None)
-        return {"country": country_found, "language": language_found}
-
-    def _generate_mandatory_search_queries(self, user_query: str, country: Optional[str], language: Optional[str]) -> List[str]:
-        searches = []
-        base = user_query.strip()
-        if country:
-            searches.append(f"{base} {country} official requirements site:gov OR site:.edu")
-            searches.append(f"{base} {country} curriculum requirements OR regulations")
-        elif language:
-            searches.append(f"{base} {language} language learning resources official exams")
-            searches.append(f"{base} {language} proficiency exam requirements OR recognized certifications")
-        else:
-            searches.append(f"{base} official guidance")
-            searches.append(f"{base} statistics OR latest data")
-        return list(set(searches))
-
-    def _detect_persona(self, query: str) -> str:
-        if not query:
-            return "Other"
-        q = query.lower()
-        if any(k in q for k in ["i am a student", "student", "grade", "class", "college", "undergraduate", "btech", "mba", "high school"]):
-            return "Student"
-        if any(k in q for k in ["i am a teacher", "teacher", "instructor", "professor", "lecturer"]):
-            return "Teacher"
-        if any(k in q for k in ["my child", "parent", "mother", "father", "guardian"]):
-            return "Parent"
-        return "Other"
-
-    def _format_structured_response(self, parsed_data: Dict[str, Any], sources_text: str, fmt: str = "structured_xml") -> str:
-        """
-        Formats parsed_data according to the requested format.
-        fmt options: structured_xml, markdown_detailed, brief_plain
-        """
-        summary = parsed_data.get('summary', '').strip()
-        explanation = parsed_data.get('explanation', "I'm sorry, I couldn't generate a detailed explanation for that.").strip()
-        insights = parsed_data.get('insights', '').strip()
-
-        if fmt == "structured_xml":
-            parts = []
-            parts.append(f"<SUMMARY>\n{summary}\n</SUMMARY>")
-            parts.append(f"<COMPREHENSIVE_EXPLANATION>\n{explanation}\n</COMPREHENSIVE_EXPLANATION>")
-            parts.append(f"<RELEVANT_INSIGHTS>\n{insights}\n</RELEVANT_INSIGHTS>")
-            if sources_text:
-                parts.append(sources_text)
-            return "\n\n".join(parts)
-
-        if fmt == "markdown_detailed":
-            out = []
-            if summary:
-                out.append(f"## Summary\n\n{summary}\n")
-            out.append(f"## Detailed Explanation\n\n{explanation}\n")
-            if insights:
-                out.append(f"## Relevant Insights\n\n{insights}\n")
-            if sources_text:
-                out.append(sources_text)
+    async def classify_intent(self, query: str) -> str:
+        """Classify the intent of the user query."""
+        # Check cache first
+        if self.cache:
+            key = f"intent_{hashlib.sha256(query.encode()).hexdigest()}"
+            cached = self.cache.get(key)
+            if cached:
+                return cached
+        
+        # Try user keyword matching
+        tokens = [t for t in re.findall(r"\b[a-zA-Z]{2,}\b", query.lower())]
+        intent_scores = Counter()
+        for intent_name, token_map in (self.user_keywords or {}).items():
+            for t in tokens:
+                intent_scores[intent_name] += token_map.get(t, 0)
+        
+        if intent_scores:
+            top_intent, top_score = intent_scores.most_common(1)[0]
+            if top_score >= 2 or (len(intent_scores) == 1 and top_score >= 1):
+                if self.cache:
+                    try:
+                        self.cache.set(key, top_intent, ttl=3600)
+                    except Exception:
+                        pass
+                return top_intent
+        
+        # Try local model
+        if self.model is not None and self.tokenizer is not None and torch is not None:
+            try:
+                self.model.eval()
+                with torch.no_grad():
+                    enc = self.tokenizer(query, return_tensors="pt", truncation=True, padding=True, max_length=128)
+                    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+                    for k, v in enc.items():
+                        enc[k] = v.to(device)
+                    self.model.to(device)
+                    outputs = self.model(**enc)
+                    logits = outputs.logits.cpu().numpy().tolist()[0]
+                    
+                    label_map = {}
+                    if self.finetune_label_map_path.exists():
+                        try:
+                            with open(self.finetune_label_map_path, "r", encoding="utf-8") as f:
+                                label_map = json.load(f)
+                        except Exception:
+                            label_map = {}
+                    
+                    if label_map:
+                        id_to_label = {int(v): k for k, v in label_map.items()}
+                        pred_idx = int(max(range(len(logits)), key=lambda i: logits[i]))
+                        intent = id_to_label.get(pred_idx, "general_guidance")
+                        if self.cache:
+                            try:
+                                self.cache.set(key, intent, ttl=3600)
+                            except Exception:
+                                pass
+                        return intent
+            except Exception:
+                logger.debug("Local classifier prediction failed; falling back to heuristics.")
+        
+        # Heuristic fallback
+        if self.intent_chain is None:
+            q = query.lower()
+            if any(w in q for w in ["resume", "cv", "cover letter"]):
+                intent = "resume_advice"
+            elif any(w in q for w in ["interview", "star method", "technical interview", "hr round"]):
+                intent = "interview_prep"
+            elif any(w in q for w in ["salary", "ctc", "package", "pay"]):
+                intent = "salary_info"
+            elif any(w in q for w in ["which course", "which college", "what should i study", "career", "i like"]):
+                intent = "career_recommendation"
+            elif any(w in q for w in ["school", "exam", "jee", "neet", "admission"]):
+                intent = "educational_guidance"
             else:
-                out.append("\n---\n*This answer is based on the model's knowledge. For up-to-date details, see the referenced sources or request a web search.*")
-            return "\n".join(out)
-
-        # brief_plain
-        if summary and len(summary) < 800:
-            short = summary
-            if explanation and len(explanation) > len(summary):
-                short += "\n\n" + (explanation if len(explanation) < 800 else explanation[:800] + "...")
-        else:
-            short = explanation if explanation else summary
-        if sources_text:
-            short += "\n\n" + sources_text
-        return short
-
-    def _get_base_prompt_template(self, intent: str, persona: str, web_context: str, rag_context: str, history_str: str, user_query: str, fmt: str = "structured_xml") -> str:
-        tone_instruction = {
-            "Student": "Adopt a motivating, clear, and actionable tone, focusing on practical steps and resources.",
-            "Teacher": "Adopt a professional, evidence-based, and resourceful tone, focusing on curriculum and pedagogy.",
-            "Parent": "Adopt an empathetic, reassuring, and organized tone, providing clear guidance.",
-            "Other": "Adopt an authoritative, informative, and comprehensive tone."
-        }.get(persona, "Adopt an authoritative, informative, and comprehensive tone.")
-
-        rag_section = ""
-        if self.rag and rag_context != "No RAG content":
-            rag_section = f"### INTERNAL KNOWLEDGE BASE (RAG):\n{rag_context}\n"
-
-        role_desc = intent.replace('_', ' ').upper()
-        if intent in ("general_guidance", "off_topic"):
-            role_desc = "GENERAL CHAT ASSISTANT (not limited to education/career)"
-
-        if fmt == "structured_xml":
-            output_instructions = """
-**MANDATORY OUTPUT STRUCTURE**:
-Your final output **MUST** use the following XML-like tags. **DO NOT** include any text outside of these tags, and **DO NOT** add your own meta-commentary, headers, or intros.
-
-<SUMMARY>A concise (2-3 sentence) summary that directly answers the user's main question. This is the "executive summary".</SUMMARY>
-
-<COMPREHENSIVE_EXPLANATION>
-This is the main, detailed answer. Provide a full explanation, address all parts of the user's query, and integrate facts from the web context. Use markdown (like lists, bolding) for clarity. This section should contain the "proof" and detailed breakdown.
-</COMPREHENSIVE_EXPLANATION>
-
-<RELEVANT_INSIGHTS>
-**This section is optional.** Only include this if you have 1-2 *highly specific, relevant* suggestions or next steps that directly follow from your explanation.
-</RELEVANT_INSIGHTS>
-"""
-        elif fmt == "markdown_detailed":
-            output_instructions = """
-Produce a clear, well-structured markdown answer with these top-level sections:
-
-## Summary
-A concise (2-3 sentence) direct answer.
-
-## Detailed Explanation
-A full answer with clear subsections, lists, bolding where appropriate, and step-by-step guidance. Cite or reference external facts when web context is provided.
-
-## Relevant Insights
-(If you have up to 1-2 highly specific next steps or suggestions, include them here.)
-
-At the end, include a "Sources" section or bullet list when web/contextual sources are available.
-"""
-        else:  # brief_plain
-            output_instructions = """
-Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 concise actionable steps. Avoid long-form exposition.
-"""
-
-        prompt = f"""
-**INSTRUCTIONS TO AI COUNSOLER**
-
-1.  **ROLE**: You are an Ultra-Advanced Hybrid Counselor and {role_desc}, specializing in {intent.replace('_', ' ').upper()} guidance.
-2.  **USER CONTEXT**: Persona: **{persona}**. {tone_instruction}
-3.  **LANGUAGE**: Respond *only* in the language of the user's query.
-4.  **CONTEXTS**:
-    {rag_section}
-    ### EXTERNAL WEB SEARCH CONTEXT (Tavily):
-    {web_context}
-    ### HISTORY:
-    {history_str}
+                intent = "general_guidance"
+            
+            if self.cache:
+                try:
+                    self.cache.set(key, intent, ttl=3600)
+                except Exception:
+                    pass
+            return intent
+        
+        # Use LLM chain
+        try:
+            response = await self._safe_llm_invoke(self.intent_chain, {"query": query})
+            intent_text = response.get("text", "") if isinstance(response, dict) else str(response)
+            intent = intent_text.strip().lower().replace(".", "")
+            valid = ["educational_guidance", "career_recommendation", "resume_advice", "interview_prep", "salary_info", "general_guidance", "off_topic"]
+            if intent not in valid:
+                intent = "general_guidance"
+            
+            if self.cache:
+                try:
+                    self.cache.set(key, intent, ttl=3600)
+                except Exception:
+                    pass
+            return intent
+        except Exception as e:
+            logger.error(f"Intent classification failed: {e}")
+            return "general_guidance"
 
-**USER QUESTION**: {user_query}
+    async def _safe_llm_invoke(self, chain, params: Dict[str, Any], max_retries: int = 2) -> Any:
+        """Safely invoke LLM chain with retries."""
+        if chain is None or self.llm is None:
+            raise RuntimeError("LLM chain or LLM not available.")
+        
+        for attempt in range(max_retries):
+            try:
+                start = time.time()
+                if hasattr(chain, "ainvoke"):
+                    res = await chain.ainvoke(params)
+                else:
+                    loop = asyncio.get_event_loop()
+                    res = await loop.run_in_executor(None, lambda: chain.invoke(params))
+                self._update_model_stats(self.current_model, True, time.time() - start)
+                return res
+            except Exception as e:
+                self._update_model_stats(self.current_model, False, error=str(e))
+                msg = str(e).lower()
+                if any(k in msg for k in ["not found", "404", "not supported", "invalid model", "model does not exist"]):
+                    if self._fallback_to_next_model():
+                        logger.info("Retrying after fallback model selection.")
+                        continue
+                    else:
+                        raise RuntimeError("All models failed.")
+                if attempt < max_retries - 1:
+                    await asyncio.sleep(2 ** attempt)
+                else:
+                    raise
 
-{output_instructions}
-"""
-        return prompt
+    # ============================================
+    # MAIN API - GET COMPREHENSIVE ANSWER
+    # ============================================
 
-    def _setup_prompts(self):
-        self.intent_chain = None
-        if self.llm and LLMChain and PromptTemplate:
-            self.intent_template = """You are an intent classifier. Respond only with one of: educational_guidance, career_recommendation, resume_advice, interview_prep, salary_info, general_guidance, off_topic. Query: {query}"""
+    async def get_comprehensive_answer(self, user_query: str, session_id: str) -> AsyncGenerator[str, None]:
+        """Generate a comprehensive, adaptively-formatted answer."""
+        
+        # Load history
+        history = []
+        try:
+            if self.db:
+                history = self.db.get_history(session_id)
+        except Exception:
+            logger.debug("History fetch failed.")
+        
+        history_str = "\n".join([f"User: {h[0]}\nBot: {h[1]}" for h in history]) if history else "No history yet."
+        
+        web_context = "No web search required or performed."
+        sources_text = ""
+        
+        normalized_query = user_query.strip()
+        normalized_lower = normalized_query.lower().rstrip(".!?")
+        
+        # Handle greetings
+        if normalized_lower in self.greetings:
+            greeting_response = "Hello! I'm your AI education & career counselor. How can I assist you with your education or career goals today?"
+            yield greeting_response
             try:
-                self.intent_chain = LLMChain(llm=self.llm, prompt=PromptTemplate.from_template(self.intent_template))
-            except Exception as e:
-                logger.error(f"Failed to create intent chain: {e}")
-                self.intent_chain = None
+                if self.db:
+                    self.db.save_history(session_id, history + [[user_query, greeting_response]])
+            except Exception:
+                pass
+            try:
+                self._learn_from_interaction(user_query, "general_guidance")
+            except Exception:
+                pass
+            return
+        
+        # Check for illegal requests
+        if self._is_illegal_request(user_query):
+            refusal = "I cannot assist with requests that enable illegal or harmful activities. I can help with lawful education, career guidance, coding practice, and study resources — please rephrase your question."
+            yield refusal
+            return
+        
+        try:
+            # Detect context
+            persona = self._detect_persona(user_query)
+            geo_lang = self._detect_country_language(user_query)
+            country = geo_lang.get("country")
+            language = geo_lang.get("language")
+            intent = await self.classify_intent(user_query)
+            
+            # ★ KEY INNOVATION: Detect query type for adaptive formatting
+            query_type = detect_query_type(user_query)
+            logger.info(f"Intent: {intent}, Query Type: {query_type}")
+            
+            # Determine if web search needed
+            use_web = self._should_use_web_search(intent, user_query, history) or (country or language)
+            
+            # Perform web search if needed
+            combined_docs: List[SimpleDoc] = []
+            if use_web:
+                search_queries: List[str] = []
+                if country or language:
+                    search_queries.extend(self._generate_mandatory_search_queries(user_query, country, language))
+                if intent == "salary_info" or self._should_use_web_search(intent, user_query):
+                    search_queries.append(f"Latest verified information for: {user_query}")
+                
+                search_queries = list(set(search_queries))
+                for sq in search_queries:
+                    docs = await self._rotate_tavily_key(sq)
+                    combined_docs.extend(docs)
+                
+                if combined_docs:
+                    unique_docs = {d.page_content: d for d in combined_docs}
+                    final_docs = list(unique_docs.values())[:10]
+                    web_context = "\n\n".join([
+                        f"Source: {getattr(doc, 'metadata', {}).get('source', 'N/A')}\n"
+                        f"Title: {getattr(doc, 'metadata', {}).get('title', '')}\n"
+                        f"Content: {getattr(doc, 'page_content', '')}"
+                        for doc in final_docs
+                    ])
+                    sources_text = format_sources_block(final_docs)
+                else:
+                    web_context = "Web search performed but returned no highly relevant results."
+                    sources_text = ""
+            
+            # Get RAG context
+            rag_context = "No RAG content"
+            if self.rag:
+                rag_context = "Local knowledge base accessed and utilized."
+            
+            # ★ Generate adaptive prompt based on query type
+            prompt = get_adaptive_prompt(
+                query_type=query_type,
+                persona=persona,
+                intent=intent,
+                web_context=web_context,
+                rag_context=rag_context,
+                history_str=history_str,
+                user_query=user_query
+            )
+            
+            # Get LLM response
+            generated_answer_text = await self._call_direct_llm(prompt)
+            
+            if not generated_answer_text:
+                raise RuntimeError("LLM returned an empty response.")
+            
+            # ★ Clean up response (remove any residual XML tags)
+            final_response = self._clean_response(generated_answer_text)
+            
+            # Add sources if available
+            if sources_text and query_type not in [QueryType.QUICK, QueryType.CONVERSATIONAL]:
+                final_response = final_response.strip() + "\n\n" + sources_text
+            
+            yield final_response
+            
+            # Save to history
+            if self.db:
+                try:
+                    self.db.save_history(session_id, history + [[user_query, final_response]])
+                except Exception:
+                    pass
+            
+            # Learn from interaction
+            try:
+                self._learn_from_interaction(user_query, intent)
+            except Exception:
+                logger.debug("Learning step failed (non-fatal).")
+        
+        except Exception as e:
+            logger.error(f"❌ Error in get_comprehensive_answer: {e}", exc_info=True)
+            error_msg = f"I'm sorry, I'm encountering an unexpected error while processing your request. Please try again."
+            if self.db:
+                try:
+                    self.db.save_history(session_id, history + [[user_query, error_msg]])
+                except Exception:
+                    pass
+            yield error_msg
 
-    # --- New: output format selection logic ---
-    def _select_output_format(self, intent: str, user_query: str, web_used: bool) -> str:
-        q = user_query.lower()
-        if any(k in q for k in ["use xml", "use tags", "provide xml", "<summary>", "with tags", "machine readable", "parseable"]):
-            return "structured_xml"
-        if "with sources" in q or "cite" in q or web_used:
-            return "markdown_detailed"
-        if intent in ("salary_info",):
-            return "markdown_detailed"
-        if len(q.split()) <= 6 and any(g in q for g in self.greetings):
-            return "brief_plain"
-        if intent in ("career_recommendation", "educational_guidance", "resume_advice", "interview_prep"):
-            return "markdown_detailed"
-        if any(k in q for k in ["how to", "step by step", "steps to", "give steps", "do this"]) or any(k in q for k in ["how do i", "how can i"]):
-            return "markdown_detailed"
-        return "markdown_detailed"
-
-    # --- New: unified parser that handles multiple formats ---
-    def _parse_response_by_format(self, fmt: str, text: str) -> Dict[str, str]:
+    def _clean_response(self, text: str) -> str:
+        """Clean up LLM response by removing XML tags and fixing formatting."""
         if not text:
-            return {"summary": "", "explanation": "", "insights": ""}
-
-        if re.search(r"<SUMMARY>(.*?)</SUMMARY>", text, re.IGNORECASE | re.DOTALL):
-            return _parse_llm_tagged_output(text)
+            return ""
+        
+        # Remove XML tags
+        text = re.sub(r'</?summary>', '', text, flags=re.IGNORECASE)
+        text = re.sub(r'</?explanation>', '', text, flags=re.IGNORECASE)
+        text = re.sub(r'</?insights>', '', text, flags=re.IGNORECASE)
+        
+        # Remove rigid section headers if they don't add value
+        text = re.sub(r'^## Summary\s*\n', '', text, flags=re.MULTILINE)
+        text = re.sub(r'^## Detailed Explanation\s*\n', '', text, flags=re.MULTILINE)
+        text = re.sub(r'^## Relevant Insights\s*\n', '', text, flags=re.MULTILINE)
+        text = re.sub(r'^Detailed Explanation\s*\n', '', text, flags=re.MULTILINE)
+        
+        # Clean up extra whitespace while preserving intentional line breaks
+        text = re.sub(r'\n{3,}', '\n\n', text)
+        
+        return text.strip()
+
+    # ============================================
+    # LEARNING AND PERSISTENCE
+    # ============================================
 
-        if fmt == "structured_xml":
-            md = _parse_markdown_structured(text)
-            return md
+    def _persist_user_keywords(self):
+        """Persist user keywords to file."""
+        try:
+            with open(self.user_keywords_path, "w", encoding="utf-8") as f:
+                json.dump(self.user_keywords, f, ensure_ascii=False, indent=2)
+        except Exception as e:
+            logger.debug(f"Failed to persist user keywords: {e}")
 
-        if fmt == "markdown_detailed":
-            md = _parse_markdown_structured(text)
-            if md.get("explanation"):
-                return md
-            return _parse_plain_text(text)
+    def _persist_greetings(self):
+        """Persist greetings to file."""
+        try:
+            with open(self.user_greetings_path, "w", encoding="utf-8") as f:
+                json.dump({"greetings": sorted(list(self.greetings))}, f, ensure_ascii=False, indent=2)
+        except Exception as e:
+            logger.debug(f"Failed to persist user greetings: {e}")
 
-        return _parse_plain_text(text)
+    def _learn_from_interaction(self, query: str, intent: str):
+        """Learn from user interaction for continuous improvement."""
+        if not query:
+            return
+        
+        q = query.strip()
+        
+        # Save to corpus
+        try:
+            with open(self.user_corpus_path, "a", encoding="utf-8") as f:
+                f.write(q + "\n")
+        except Exception:
+            pass
+        
+        # Extract and save keywords
+        tokens = [t for t in re.findall(r"\b[a-zA-Z]{2,}\b", q.lower()) if len(t) > 1]
+        
+        if len(tokens) <= 2 and q.lower() not in {"", "ok", "thanks", "thank you"}:
+            if any(g in q.lower() for g in ["hi", "hello", "hey", "hlo", "hiii", "hii"]):
+                self.greetings.add(q.lower())
+                self._persist_greetings()
+        
+        if intent not in self.user_keywords:
+            self.user_keywords[intent] = {}
+        
+        token_counts = self.user_keywords.get(intent, {})
+        for t in tokens:
+            token_counts[t] = token_counts.get(t, 0) + 1
+        self.user_keywords[intent] = token_counts
+        self._persist_user_keywords()
+        
+        # Persist fine-tune example
+        try:
+            self._persist_fine_tune_example(q, intent)
+        except Exception:
+            logger.debug("Failed to persist fine-tune example (non-fatal).")
 
-    # --- New: fine-tune persistence helpers (unchanged) ---
     def _persist_fine_tune_example(self, text: str, label: str) -> None:
+        """Persist fine-tune example to Hugging Face dataset."""
         try:
             line = json.dumps({"text": text, "label": label}, ensure_ascii=False)
             with open(self.local_examples_path, "a", encoding="utf-8") as f:
                 f.write(line + "\n")
-
+            
             api = HfApi()
             api.upload_file(
                 path_or_fileobj=self.local_examples_path,
@@ -882,6 +1357,7 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
             logger.debug(f"Failed to persist fine-tune example to Hub: {e}")
 
     def _load_fine_tune_examples(self) -> List[Dict[str, str]]:
+        """Load fine-tune examples from Hugging Face."""
         try:
             hf_hub_download(
                 repo_id=self.dataset_repo_id,
@@ -891,11 +1367,11 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
                 token=os.environ.get("HF_WRITE_TOKEN"),
                 force_filename=self.examples_filename_in_repo
             )
-
+            
             if not self.local_examples_path.exists():
                 logger.info("No examples file found in dataset repo.")
                 return []
-
+            
             with open(self.local_examples_path, "r", encoding="utf-8") as f:
                 lines = [json.loads(l) for l in f if l.strip()]
             return lines
@@ -904,6 +1380,7 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
             return []
 
     def _clear_fine_tune_examples(self, archive: bool = True):
+        """Clear fine-tune examples (optionally archive first)."""
         api = HfApi()
         try:
             if archive:
@@ -923,19 +1400,23 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
                     repo_type="dataset",
                     token=os.environ.get("HF_WRITE_TOKEN")
                 )
-
+            
             for f in glob.glob(f"./{self.examples_filename_in_repo}*"):
                 try:
                     os.remove(f)
                 except Exception:
                     pass
+            
             logger.info("Archived examples file in dataset repo.")
-
         except Exception as e:
             logger.debug(f"Failed to clear/archive examples in Hub (non-fatal): {e}")
 
-    # --- Background fine-tune loop (unchanged) ---
+    # ============================================
+    # FINE-TUNE WORKER
+    # ============================================
+
     def _fine_tune_loop_sync(self):
+        """Background fine-tuning loop."""
         logger.info("Fine-tune loop running.")
         while not getattr(self, "_stop_fine_tune_worker", False):
             try:
@@ -945,19 +1426,22 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
             time.sleep(max(10, self.fine_tune_interval))
 
     def _maybe_fine_tune_once(self):
+        """Attempt a fine-tuning iteration if conditions are met."""
         if not self._fine_tune_lock.acquire(blocking=False):
             logger.debug("Fine-tune run already in progress; skipping this iteration.")
             return
+        
         try:
             examples = self._load_fine_tune_examples()
             if len(examples) < self.min_examples_to_train:
                 logger.debug(f"Not enough examples for fine-tune (have {len(examples)}, need {self.min_examples_to_train}).")
                 return
-
+            
             if not (torch and self.model is not None and self.tokenizer is not None):
                 logger.warning("Fine-tune prerequisites missing (torch/model/tokenizer). Skipping training.")
                 return
-
+            
+            # Build label map
             label_to_id = {}
             if self.label_encoder is not None and hasattr(self.label_encoder, "classes_"):
                 for idx, lab in enumerate(getattr(self.label_encoder, "classes_", [])):
@@ -969,40 +1453,40 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
                             label_to_id = json.load(f)
                     except Exception:
                         label_to_id = {}
-
+            
             next_id = max(label_to_id.values()) + 1 if label_to_id else 0
             for ex in examples:
                 lab = ex.get("label", "general_guidance")
                 if lab not in label_to_id:
                     label_to_id[lab] = next_id
                     next_id += 1
-
+            
             try:
                 with open(self.finetune_label_map_path, "w", encoding="utf-8") as f:
                     json.dump(label_to_id, f, ensure_ascii=False, indent=2)
             except Exception:
                 pass
-
+            
+            # Prepare data
             texts = [ex["text"] for ex in examples]
             labels = [label_to_id.get(ex.get("label", "general_guidance"), 0) for ex in examples]
-
+            
             enc = self.tokenizer(texts, padding=True, truncation=True, max_length=128, return_tensors="pt")
             input_ids = enc["input_ids"]
             attention_mask = enc["attention_mask"]
             labels_tensor = torch.tensor(labels, dtype=torch.long)
-
+            
             dataset = TensorDataset(input_ids, attention_mask, labels_tensor)
             sampler = RandomSampler(dataset)
             loader = DataLoader(dataset, sampler=sampler, batch_size=self.fine_tune_batch_size)
-
+            
             device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
             self.model.to(device)
             self.model.train()
             optimizer = AdamW(self.model.parameters(), lr=1e-5)
-
-            total_steps = len(loader) * max(1, self.fine_tune_epochs)
+            
             logger.info(f"Starting fine-tune: {len(examples)} examples, {len(loader)} batches, epochs={self.fine_tune_epochs}")
-
+            
             for epoch in range(self.fine_tune_epochs):
                 epoch_loss = 0.0
                 for batch in loader:
@@ -1016,7 +1500,8 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
                     optimizer.step()
                     epoch_loss += loss.item() if loss is not None else 0.0
                 logger.info(f"Fine-tune epoch {epoch+1}/{self.fine_tune_epochs} loss: {epoch_loss:.4f}")
-
+            
+            # Save model
             try:
                 self.model.save_pretrained(self.model_path)
                 try:
@@ -1026,266 +1511,30 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
                 logger.info(f"✅ Fine-tuned model saved to {self.model_path}")
             except Exception as e:
                 logger.error(f"Failed to save fine-tuned model: {e}")
-
+            
             self._clear_fine_tune_examples(archive=True)
-
+        
         finally:
             try:
                 self._fine_tune_lock.release()
             except Exception:
                 pass
 
-    # --- Lightweight learning helpers (unchanged) ---
-    def _persist_user_keywords(self):
-        try:
-            with open(self.user_keywords_path, "w", encoding="utf-8") as f:
-                json.dump(self.user_keywords, f, ensure_ascii=False, indent=2)
-        except Exception as e:
-            logger.debug(f"Failed to persist user keywords: {e}")
-
-    def _persist_greetings(self):
-        try:
-            with open(self.user_greetings_path, "w", encoding="utf-8") as f:
-                json.dump({"greetings": sorted(list(self.greetings))}, f, ensure_ascii=False, indent=2)
-        except Exception as e:
-            logger.debug(f"Failed to persist user greetings: {e}")
-
-    def _learn_from_interaction(self, query: str, intent: str):
-        if not query:
-            return
-        q = query.strip()
-        try:
-            with open(self.user_corpus_path, "a", encoding="utf-8") as f:
-                f.write(q + "\n")
-        except Exception:
-            pass
-
-        tokens = [t for t in re.findall(r"\b[a-zA-Z]{2,}\b", q.lower()) if len(t) > 1]
-        if len(tokens) <= 2 and q.lower() not in {"", "ok", "thanks", "thank you"}:
-            if any(g in q.lower() for g in ["hi", "hello", "hey", "hlo", "hiii", "hii"]):
-                self.greetings.add(q.lower())
-                self._persist_greetings()
-
-        if intent not in self.user_keywords:
-            self.user_keywords[intent] = {}
-        token_counts = self.user_keywords.get(intent, {})
-        for t in tokens:
-            token_counts[t] = token_counts.get(t, 0) + 1
-        self.user_keywords[intent] = token_counts
-        self._persist_user_keywords()
-
-        try:
-            self._persist_fine_tune_example(q, intent)
-        except Exception:
-            logger.debug("Failed to persist fine-tune example (non-fatal).")
-
-    # --- Core API Methods ---
-    async def get_comprehensive_answer(self, user_query: str, session_id: str) -> AsyncGenerator[str, None]:
-        history = []
-        try:
-            if self.db:
-                history = self.db.get_history(session_id)
-        except Exception:
-            logger.debug("History fetch failed.")
-
-        history_str = "\n".join([f"User: {h[0]}\nBot: {h[1]}" for h in history]) if history else "No history yet."
-        web_context = "No web search required or performed."
-        sources_text = ""
-
-        normalized_query = user_query.strip()
-        normalized_lower = normalized_query.lower().rstrip(".!?")
-        greetings = set(self.greetings)
-
-        if normalized_lower in greetings:
-            greeting_response = "Hello! I'm your AI education & career counselor. How can I assist you with your education or career goals today?"
-            yield greeting_response
-            try:
-                if self.db:
-                    self.db.save_history(session_id, history + [[user_query, greeting_response]])
-            except Exception:
-                pass
-            try:
-                self._learn_from_interaction(user_query, "general_guidance")
-            except Exception:
-                pass
-            return
-
-        if self._is_illegal_request(user_query):
-            refusal = "I cannot assist with requests that enable illegal or harmful activities. I can help with lawful education, career guidance, coding practice, and study resources — please rephrase your question."
-            yield refusal
-            return
-
-        try:
-            persona = self._detect_persona(user_query)
-            geo_lang = self._detect_country_language(user_query)
-            country = geo_lang.get("country")
-            language = geo_lang.get("language")
-
-            intent = await self.classify_intent(user_query)
-            logger.info(f"Intent detected: {intent}")
-
-            use_web = self._should_use_web_search(intent, user_query, history) or (country or language)
-            combined_docs: List[SimpleDoc] = []
-
-            if use_web:
-                search_queries: List[str] = []
-                if country or language:
-                    search_queries.extend(self._generate_mandatory_search_queries(user_query, country, language))
-                if intent == "salary_info" or self._should_use_web_search(intent, user_query):
-                    search_queries.append(f"Latest verified information for: {user_query}")
-
-                search_queries = list(set(search_queries))
-
-                for sq in search_queries:
-                    docs = await self._rotate_tavily_key(sq)
-                    combined_docs.extend(docs)
-
-                if combined_docs:
-                    unique_docs = {d.page_content: d for d in combined_docs}
-                    final_docs = list(unique_docs.values())[:10]
-
-                    web_context = "\n\n".join([f"Source: {getattr(doc, 'metadata', {}).get('source','N/A')}\nTitle: {getattr(doc, 'metadata', {}).get('title','')}\nContent: {getattr(doc, 'page_content','')}" for doc in final_docs])
-                    sources_text = format_sources_block(final_docs)
-                else:
-                    web_context = "Web search performed but returned no highly relevant results."
-                    sources_text = "No reliable external sources were found for this query."
-
-            rag_context = "No RAG content"
-            if self.rag:
-                rag_context = "Local knowledge base accessed and utilized."
-
-            fmt = self._select_output_format(intent, user_query, use_web)
-            prompt = self._get_base_prompt_template(intent, persona, web_context, rag_context, history_str, user_query, fmt=fmt)
-
-            generated_answer_text = await self._call_direct_llm(prompt)
-
-            if not generated_answer_text:
-                raise RuntimeError("LLM returned an empty response.")
-
-            parsed_data = self._parse_response_by_format(fmt, generated_answer_text)
-            structured_response = self._format_structured_response(parsed_data, sources_text, fmt=fmt)
-            yield structured_response
-
-            if self.db:
-                try:
-                    self.db.save_history(session_id, history + [[user_query, structured_response]])
-                except Exception:
-                    pass
-
-            try:
-                self._learn_from_interaction(user_query, intent)
-            except Exception:
-                logger.debug("Learning step failed (non-fatal).")
-
-        except Exception as e:
-            logger.error(f"❌ Error in get_comprehensive_answer: {e}", exc_info=True)
-            error_msg = f"I'm sorry, I'm encountering an unexpected error while processing your request. Current model: {self.current_model}. Please try again."
-            if self.db:
-                try:
-                    self.db.save_history(session_id, history + [[user_query, error_msg]])
-                except Exception:
-                    pass
-            yield error_msg
-
-    async def classify_intent(self, query: str) -> str:
-        if self.cache:
-            key = f"intent_{hashlib.sha256(query.encode()).hexdigest()}"
-            cached = self.cache.get(key)
-            if cached:
-                return cached
-
-        tokens = [t for t in re.findall(r"\b[a-zA-Z]{2,}\b", query.lower())]
-        intent_scores = Counter()
-        for intent_name, token_map in (self.user_keywords or {}).items():
-            for t in tokens:
-                intent_scores[intent_name] += token_map.get(t, 0)
-        if intent_scores:
-            top_intent, top_score = intent_scores.most_common(1)[0]
-            if top_score >= 2 or (len(intent_scores) == 1 and top_score >= 1):
-                if self.cache:
-                    try:
-                        self.cache.set(key, top_intent, ttl=3600)
-                    except Exception:
-                        pass
-                return top_intent
-
-        if self.model is not None and self.tokenizer is not None and torch is not None:
-            try:
-                self.model.eval()
-                with torch.no_grad():
-                    enc = self.tokenizer(query, return_tensors="pt", truncation=True, padding=True, max_length=128)
-                    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-                    for k, v in enc.items():
-                        enc[k] = v.to(device)
-                    self.model.to(device)
-                    outputs = self.model(**enc)
-                    logits = outputs.logits.cpu().numpy().tolist()[0]
-                    label_map = {}
-                    if self.finetune_label_map_path.exists():
-                        try:
-                            with open(self.finetune_label_map_path, "r", encoding="utf-8") as f:
-                                label_map = json.load(f)
-                        except Exception:
-                            label_map = {}
-                    if label_map:
-                        id_to_label = {int(v): k for k, v in label_map.items()}
-                        pred_idx = int(max(range(len(logits)), key=lambda i: logits[i]))
-                        intent = id_to_label.get(pred_idx, "general_guidance")
-                        if self.cache:
-                            try:
-                                self.cache.set(key, intent, ttl=3600)
-                            except Exception:
-                                pass
-                        return intent
-            except Exception:
-                logger.debug("Local classifier prediction failed; falling back to heuristics.")
-
-        if self.intent_chain is None:
-            q = query.lower()
-            if any(w in q for w in ["resume", "cv", "cover letter"]):
-                intent = "resume_advice"
-            elif any(w in q for w in ["interview", "star method", "technical interview", "hr round"]):
-                intent = "interview_prep"
-            elif any(w in q for w in ["salary", "ctc", "package", "pay"]):
-                intent = "salary_info"
-            elif any(w in q for w in ["which course", "which college", "what should i study", "career", "i like"]):
-                intent = "career_recommendation"
-            elif any(w in q for w in ["school", "exam", "jee", "neet", "admission"]):
-                intent = "educational_guidance"
-            else:
-                intent = "general_guidance"
-            if self.cache:
-                try:
-                    self.cache.set(key, intent, ttl=3600)
-                except Exception:
-                    pass
-            return intent
-
-        try:
-            response = await self._safe_llm_invoke(self.intent_chain, {"query": query})
-            intent_text = response.get("text", "") if isinstance(response, dict) else str(response)
-            intent = intent_text.strip().lower().replace(".", "")
-            valid = ["educational_guidance", "career_recommendation", "resume_advice", "interview_prep", "salary_info", "general_guidance", "off_topic"]
-            if intent not in valid:
-                intent = "general_guidance"
-            if self.cache:
-                try:
-                    self.cache.set(key, intent, ttl=3600)
-                except Exception:
-                    pass
-            return intent
-        except Exception as e:
-            logger.error(f"Intent classification failed: {e}")
-            return "general_guidance"
+    # ============================================
+    # CAREER PREDICTION
+    # ============================================
 
     async def predict_career(self, query: str) -> Dict[str, Any]:
+        """Predict career recommendation based on query."""
         if self.cache:
             key = f"predict_{hashlib.sha256(query.encode()).hexdigest()}"
             cached = self.cache.get(key)
             if cached:
                 return cached
+        
         if not (self.model and self.tokenizer and torch and self.label_encoder is not None):
             return {"recommendation": None, "confidence": 0.0, "error": "Local prediction unavailable"}
+        
         try:
             inputs = self.tokenizer(query.lower(), return_tensors="pt", padding=True, truncation=True, max_length=128)
             with torch.no_grad():
@@ -1302,8 +1551,12 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
             logger.error(f"Prediction failed: {e}")
             return {"recommendation": None, "confidence": 0.0, "error": str(e)}
 
-    # --- Health and Info Getters ---
+    # ============================================
+    # HEALTH AND INFO
+    # ============================================
+
     def get_current_model_info(self) -> Dict[str, Any]:
+        """Get current model information."""
         return {
             "current_model": self.current_model,
             "available_models": self.available_models,
@@ -1315,6 +1568,7 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
         }
 
     def get_health_status(self) -> Dict[str, Any]:
+        """Get system health status."""
         try:
             total_models = len(self.available_models)
             working = sum(1 for s in self.model_performance_stats.values() if s.get("success_rate", 0) > 0)
@@ -1332,31 +1586,34 @@ Produce a short, conversational reply (1-4 sentences). If helpful, include 1-2 c
         except Exception as e:
             return {"status": "error", "error": str(e), "last_updated": time.time()}
 
+
+# ============================================
+# DEMO
+# ============================================
+
 if __name__ == "__main__":
     async def demo():
         c = UltraAdvancedHybridCounselor()
-        q1 = "What is the average starting salary for a full-stack developer in San Francisco, USA, and what steps should I take to prepare for interviews? I am a student."
-        print("\n---- Query 1 (Web search + Persona + Geo) ----")
-        async for out in c.get_comprehensive_answer(q1, session_id="demo_advanced"):
-            print(out)
-        q2 = "what is data science"
-        print("\n---- Query 2 (Simple definition) ----")
-        async for out in c.get_comprehensive_answer(q2, session_id="demo_simple"):
-            print(out)
-        q3 = "How to make a bomb at home"
-        print("\n---- Query 3 (Illegal -> refused) ----")
-        async for out in c.get_comprehensive_answer(q3, session_id="demo_illegal"):
-            print(out)
-        q4 = "Help me explore different career paths based on my interests and skills"
-        print("\n---- Query 4 (LLM preferred) ----")
-        async for out in c.get_comprehensive_answer(q4, session_id="demo_basic"):
-            print(out)
-        q5 = "hlo"
-        print("\n---- Query 5 (Greeting -> learned) ----")
-        async for out in c.get_comprehensive_answer(q5, session_id="demo_greet"):
-            print(out)
-
+        
+        test_queries = [
+            ("What is machine learning?", "Quick answer test"),
+            ("Give me syllabus for machine learning", "Syllabus test"),
+            ("How to become a data scientist?", "How-to test"),
+            ("Python vs JavaScript for web development", "Comparison test"),
+            ("Give me roadmap for becoming a full stack developer", "Roadmap test"),
+            ("Top 5 programming languages to learn in 2025", "List test"),
+        ]
+        
+        for query, desc in test_queries:
+            print(f"\n{'='*60}")
+            print(f"TEST: {desc}")
+            print(f"Query: {query}")
+            print(f"{'='*60}")
+            async for out in c.get_comprehensive_answer(query, session_id="demo"):
+                print(out)
+            print()
+    
     try:
         asyncio.run(demo())
     except Exception as e:
-        logger.error(f"Demo failed: {e}")
+        logger.error(f"Demo failed: {e}")