app/services/rag_pipeline.py

import logging
import hashlib
import re
from collections import OrderedDict
from typing import Dict, List, Tuple

from app.services.llm import LLMService
from app.services.vector_store import FaissVectorStore
from app.services.reranker import RerankerService

logger = logging.getLogger(__name__)

# ── In-memory LRU answer cache ──────────────────────────────────────────────
# Keyed on MD5 of the normalized message. Holds up to 128 unique answers.
# Clears automatically on app restart (intentional — KB could be re-indexed).
_CACHE_MAX = 128
_answer_cache: OrderedDict = OrderedDict()


def _cache_key(message: str) -> str:
    """Returns a stable hash key for a normalized message string."""
    return hashlib.md5(message.lower().strip().encode()).hexdigest()


# ── Local intent-based query expander ────────────────────────────────────────
# Deterministic Python rules — runs in microseconds, saves 3–6s per request.
# IMPORTANT: ordered most-specific → least-specific to prevent false matches.
# On first match, BREAKS — only one intent applied per query.
_INTENT_MAP: List[tuple] = [
    # ── Leave types (most specific first) ──
    # ── Broad "all leaves" patterns (must come BEFORE specific types) ──
    ("per annum",     ["casual leave sick leave annual leave maternity paternity hajj bereavement study unauthorized",
                       "all leave types employee entitlement days count per annum"]),
    ("total leave",   ["casual leave sick leave annual leave maternity paternity hajj bereavement study unauthorized",
                       "total leave types employee entitlement days count"]),
    ("how many leave",["casual leave sick leave annual leave maternity paternity hajj bereavement study unauthorized",
                       "all leave types employee entitlement days count"]),
    ("kitni leave",   ["casual leave sick leave annual leave maternity paternity hajj bereavement study unauthorized",
                       "all leave types employee entitlement days count"]),
    ("kitni chutti",  ["casual leave sick leave annual leave maternity paternity hajj bereavement study unauthorized",
                       "all leave types employee entitlement days count"]),
    ("paid leave",    ["casual leave sick leave annual leave maternity paternity hajj bereavement study unauthorized",
                       "paid leave types employee entitlement days count"]),
    ("all leave",     ["casual leave sick leave annual leave maternity paternity hajj bereavement study unauthorized",
                       "all leave types employee entitlement days count"]),
    ("maternity",     ["maternity leave days duration policy", "paid leave maternity employee pregnancy benefit"]),
    ("paternity",     ["paternity leave days duration policy", "paid leave paternity employee childbirth"]),
    ("hajj",          ["hajj leave days duration policy", "paid leave hajj religious pilgrimage"]),
    ("bereavement",   ["bereavement leave death family days", "compassionate leave policy"]),
    ("sick leave",    ["sick leave days count policy", "medical leave employee entitlement"]),
    ("casual leave",  ["casual leave days count policy", "leave types employee"]),
    ("annual leave",  ["annual leave days count policy", "leave entitlement per year carry forward"]),
    ("study leave",   ["study leave education policy days", "employee study leave entitlement"]),
    ("encash",        ["leave encashment casual annual department eligibility"]),
    ("unauthori",     ["unauthorized leave absence AWOL disciplinary notice penalty"]),
    ("absent",        ["absence without pay unauthorized leave AWOL salary deduction"]),
    ("leave",         ["casual leave sick leave annual leave maternity paternity hajj bereavement study unauthorized",
                       "leave types employee entitlement days"]),
    ("chutti",        ["casual leave sick leave annual leave holiday days off"]),
    # ── Office timing / Attendance ──
    ("office hour",   ["office working hours schedule", "workday start end time shift"]),
    ("work hour",     ["office working hours schedule", "workday start end time shift"]),
    ("timing",        ["office working hours schedule", "workday start end time shift hours"]),
    ("schedule",      ["office working hours schedule", "workday start end time"]),
    ("late",          ["late coming attendance 20 minutes half day deduction arrival", "shift start time tardiness"]),
    ("early out",     ["early out shift verbal approval supervising authority deduction"]),
    ("half day",      ["half day 2 hours shift start attendance policy", "six and half hours workday"]),
    ("punch",         ["time clock time card portal attendance record", "timing in timing out"]),
    ("shift",         ["work shift change approval supervising authority morning evening", "shift hours schedule"]),
    ("lunch",         ["lunch dinner break 1 hour morning evening shift"]),
    ("break",         ["smoking breaks 3 breaks workday 5 minutes lunch dinner"]),
    ("saturday",      ["saturday half day first saturday off workweek schedule"]),
    # ── Salary / Pay (compound first) ──
    ("salary increment",["salary increment raise confirmation review performance appraisal",
                         "revised salary effective date annual raise percentage"]),
    ("salary",        ["salary structure payroll compensation amount deduction", "monthly pay increment"]),
    ("advance",       ["advance salary loan request limit deduction eligibility committee"]),
    ("loan",          ["loans advance salary request limit deduction eligibility committee 6 months"]),
    ("pay",           ["salary structure payroll compensation", "payment date schedule payday"]),
    ("payroll",       ["salary payroll structure compensation", "monthly pay deduction June December"]),
    ("compensation",  ["salary compensation structure payroll", "monthly pay amount"]),
    ("deduction",     ["salary deduction per day absent leave without pay", "payroll deduction"]),
    ("increment",     ["salary increment raise confirmation review performance", "revised salary effective date"]),
    # ── Benefits / Allowances ──
    ("allowance",     ["allowances perks medical bonuses fuel transport reimbursements", "employee benefits"]),
    ("benefit",       ["allowances perks medical bonuses reimbursements", "employee benefits privileges maternity"]),
    ("perk",          ["employee perks benefits extras privileges", "allowances bonuses"]),
    ("fuel",          ["fuel allowance transport reimbursement conveyance petrol"]),
    ("medical",       ["medical allowance health insurance coverage", "medical benefits employee"]),
    ("transport",     ["transport allowance fuel reimbursement conveyance"]),
    ("bonus",         ["bonus performance incentive referral program PKR 5000"]),
    ("insurance",     ["healthcare insurance hospitalization self spouse panel hospital", "NJI claim health card"]),
    ("hospital",      ["healthcare insurance hospitalization panel non-panel emergency", "NJI claim reimbursement"]),
    ("eobi",          ["EOBI old age benefit pension contribution employer employee"]),
    ("pension",       ["EOBI old age benefit pension survivor invalidity"]),
    ("provident",     ["provident fund investment saving employer employee retirement"]),
    ("referral",      ["referral program PKR 5000 bonus qualified candidate joining"]),
    ("game room",     ["game room entertainment lunchtime room 301 male female schedule"]),
    ("game",          ["game room entertainment lunchtime room 301"]),
    # ── Contact / Portal / Tickets ──
    ("contact",       ["trouble ticket portal HR department networking admin", "problem resolution grievance"]),
    ("number",        ["trouble ticket portal HR department contact", "problem resolution"]),
    ("email",         ["trouble ticket portal HR department contact", "problem resolution"]),
    ("ticket",        ["trouble ticket portal submit view open closed department", "problem resolution urgent"]),
    ("portal",        ["trouble ticket portal submit HR networking admin", "time clock attendance"]),
    ("complain",      ["grievance procedure complaint trouble ticket portal", "supervising authority feedback form"]),
    ("issue",         ["trouble ticket portal problem resolution HR networking admin", "grievance complaint"]),
    ("problem",       ["trouble ticket portal problem resolution HR networking admin", "grievance complaint"]),
    # ── Management/situational ──
    ("reject",        ["management rights refuse reject cancel revoke authority discretion"]),
    ("refuse",        ["management rights refuse reject cancel revoke authority discretion"]),
    ("cancel",        ["management rights refuse reject cancel revoke authority discretion"]),
    ("manager",       ["management rights authority discretion approval rejection supervising authority"]),
    ("boss",          ["supervising authority management approval rejection discretion"]),
    ("supervisor",    ["supervising authority management approval rejection discretion"]),
    ("inform",        ["notification obligation inform hr manager notice period reporting"]),
    ("notify",        ["notification obligation inform hr manager notice period reporting"]),
    ("what if",       ["consequences non-compliance violation failure to inform unauthorized"]),
    # ── Termination / Resignation ──
    ("terminat",      ["termination resignation separation procedure process steps notice period", "exit policy probation confirmation dismissal"]),
    ("resign",        ["resignation separation procedure steps notice period final settlement", "termination exit process employee"]),
    ("fired",         ["termination dismissal separation misconduct level 3 immediate"]),
    ("quit",          ["resignation separation procedure notice period final settlement exit"]),
    ("notice period", ["notice period resignation termination duration days separation", "exit process managerial roles 30 60 90 days"]),
    ("notice",        ["notice period resignation termination duration days separation", "exit process managerial roles 30 60 90 days"]),
    ("exit interview",["exit interview supervisor separation HR department feedback"]),
    ("prorat",        ["pro-rata leaves salary final settlement confirmation probation days worked", "leave encashment pro-rata calculation separation"]),
    ("pro-rata",      ["pro-rata leaves salary final settlement confirmation probation days worked", "leave encashment pro-rata calculation separation"]),
    ("settlement",    ["final settlement salary adjustments loans pro-rata leaves separation", "before after salary processed"]),
    ("return of",     ["return of property identification health insurance card vehicle separation"]),
    # ── Probation / Confirmation / Permanent (compound first) ──
    ("after probation",["confirmation confirmed benefits leaves entitlement salary increment permanent",
                        "post probation confirmation 90 days employee rights"]),
    ("after confirm",  ["confirmation confirmed benefits leaves entitlement salary increment permanent",
                        "post confirmation employee rights benefits entitlement"]),
    ("probat",        ["probation period 90 days evaluation initial confirmation confirmed", "orientation training supervising authority performance dismissal employment"]),
    ("permanent",     ["confirmation confirmed probation period 90 days evaluation performance", "permanent employee orientation initial evaluation supervising authority"]),
    ("confirm",       ["confirmation confirmed probation period 90 days evaluation performance", "salary increment pro-rata leaves permanent employee"]),
    ("evaluation",    ["initial evaluation period 90 days probation performance confirmation", "supervising authority confirmed dismissed"]),
    ("pakka",         ["confirmation confirmed probation period 90 days evaluation performance"]),
    # ── Work from Home ──
    ("remote",        ["remote work work from home WFH policy telecommute emergency"]),
    ("wfh",           ["work from home evaluation percentage hours remote policy 30 70"]),
    ("work from home",["work from home policy approval supervising authority emergency 30 70 percent"]),
    # ── Discipline / Conduct / Rules ──
    ("acceptab",      ["acceptable conduct behavior policy workplace standards code of conduct", "employee rules work rules compliance"]),
    ("discipline",    ["disciplinary action policy procedure employee level 1 2 3 warning"]),
    ("warning",       ["disciplinary warning notice level 1 2 3 counseling suspension"]),
    ("suspend",       ["suspension disciplinary action level 2 3 procedure"]),
    ("code of conduct",["code of conduct policy employee behaviour rules work rules"]),
    ("harass",        ["harassment policy complaint sexual verbal physical discrimination", "supervising authority retaliation investigation"]),
    ("bully",         ["harassment bullying policy complaint workplace"]),
    # ── Confidentiality / Non-Compete ──
    ("confidential",  ["confidentiality policy client information intellectual property prohibited"]),
    ("non-compete",   ["restraint of trade non-compete cooling off period 24 months"]),
    ("moonlight",     ["activity permission outside work conflict of interest second job"]),
    ("side job",      ["activity permission outside work conflict of interest"]),
    # ── Other HR topics ──
    ("overtime",      ["overtime compensation extra hours payment policy"]),
    ("appraisal",     ["performance appraisal review increment salary raise"]),
    ("attendance",    ["attendance policy punctuality late arrival absenteeism time clock"]),
    ("dress code",    ["dress code uniform attire professional clothing policy personal appearance"]),
    ("smoking",       ["tobacco smoking breaks smoke-free workplace prohibited"]),
    ("cigarette",     ["tobacco smoking breaks smoke-free workplace prohibited"]),
    ("phone",         ["cell phone usage personal purposes silent vibrate prohibited"]),
    ("mobile",        ["cell phone usage personal purposes silent vibrate prohibited"]),
    ("computer",      ["computer email internet policy electronic communication", "trouble ticket portal networking"]),
    ("internet",      ["computer email internet policy electronic communication prohibited websites"]),
    ("usb",           ["USB prohibited office computer system administrator"]),
    ("grievance",     ["grievance complaint procedure policy supervising authority portal feedback form"]),
    ("vehicle",       ["company maintained vehicle car driving maintenance accident insurance"]),
    ("car",           ["company maintained vehicle driving maintenance accident insurance"]),
    ("holiday",       ["holidays holiday pay federal compensatory leave allowance"]),
    ("at will",       ["employment at will terminate resign any time any reason"]),
    ("copyright",     ["copyright violation ownership work intellectual property prohibited"]),
    ("data theft",    ["data theft stealing digital records company property prohibited"]),
    ("orientation",   ["orientation training initial evaluation period 90 days probation"]),
    # ── Meta / Document queries ──
    ("update",        ["updated revised version date employee handbook policy effective",
                       "MartechSol handbook revision date year"]),
    ("version",       ["updated revised version date employee handbook policy effective",
                       "MartechSol handbook revision date year"]),
    ("revised",       ["updated revised version date employee handbook policy effective",
                       "MartechSol handbook revision date year"]),
    # ── Broad overview / Meta queries ──
    ("polic",         ["employee benefits leaves salary attendance work rules disciplinary procedures", "workplace orientation separation confirmation grievance"]),
    ("rule",          ["work rules disciplinary procedures level 1 2 3 employee responsibilities", "attendance leave policy code of conduct"]),
    ("apply",         ["trouble ticket portal HR department contact", "leave application process approval"]),
    ("job",           ["trouble ticket portal HR department contact"]),
    ("hiring",        ["trouble ticket portal HR department contact"]),
    ("recruit",       ["trouble ticket portal HR department contact"]),
]


def _expand_query_locally(message: str) -> List[str]:
    """
    Expands a user query into targeted search strings using keyword rules.
    Deterministic, runs in microseconds, zero API cost.
    On first matching intent, BREAKS — only one intent per query for precision.
    """
    msg_lower = message.lower()
    queries: List[str] = [message]  # Original query always first

    for keyword, variants in _INTENT_MAP:
        if keyword in msg_lower:
            for v in variants:
                if v not in queries:
                    queries.append(v)
            break  # Only apply first (most specific) matching intent

    return queries[:3]  # Cap at 3, consistent with proven behaviour


# ── Conversational Follow-Up Resolver ────────────────────────────────────────
# Resolves pronouns like "them", "it", "this" in follow-up queries using the
# last user message from history. Used ONLY for retrieval — the LLM gets the
# original message + history for natural conversation flow.
_FOLLOWUP_PRONOUNS = {"them", "then", "it", "this", "that", "those", "these", "its"}
_FOLLOWUP_STARTERS = {
    "and ", "also ", "what about ", "how about ", "how to ",
    # Prepositional modifiers — almost always continue the previous topic
    # e.g. "during probation" after "leaves per annum" = "leaves during probation"
    "during ", "after ", "before ", "for ", "without ", "on ", "in ", "if ",
}


def _resolve_followup(message: str, history: List[Dict[str, str]]) -> str:
    """Resolves follow-up pronouns for better retrieval.
    e.g., 'how to get them' + history['sick leaves'] → 'how to get sick leaves'
    Only affects retrieval — LLM gets original message + full history."""
    if not history:
        return message

    msg_lower = message.lower().strip()
    words = set(msg_lower.split())

    # Detect follow-up indicators
    has_pronoun = bool(words & _FOLLOWUP_PRONOUNS)
    is_short_followup = len(words) <= 5 and any(msg_lower.startswith(s) for s in _FOLLOWUP_STARTERS)

    # Also detect ultra-short queries with no topic keywords (e.g. "how to get" after "sick leaves")
    has_topic = any(kw in msg_lower for kw, _ in _INTENT_MAP)
    is_vague_query = len(words) <= 5 and not has_topic and not _is_greeting(message)

    if not has_pronoun and not is_short_followup and not is_vague_query:
        return message  # Not a follow-up — use as-is

    # Find the last user message for context
    last_user_msg = None
    for msg in reversed(history):
        if msg.get("role") == "user":
            last_user_msg = msg["content"].strip()
            break

    if not last_user_msg:
        return message

    # For pronoun-containing queries: replace first matching pronoun with last topic
    if has_pronoun:
        for pronoun in _FOLLOWUP_PRONOUNS:
            pattern = re.compile(r'\b' + pronoun + r'\b', re.IGNORECASE)
            if pattern.search(message):
                resolved = pattern.sub(last_user_msg, message, count=1)
                logger.info("Follow-up resolved: '%s' → '%s'", message, resolved)
                return resolved

    # For very short follow-ups like "and paternity?", prepend context
    if is_short_followup or is_vague_query:
        resolved = f"{last_user_msg} {message}"
        logger.info("Follow-up prepended: '%s' → '%s'", message, resolved)
        return resolved

    return message


def _is_greeting(message: str) -> bool:
    """Detects greetings, casual chat, rude/dismissive, thanks, bye — skip retrieval for these.
    These go straight to LLM with conversational intelligence (no Expert Data needed)."""
    
    msg_lower = message.lower().strip()
    words = msg_lower.split()
    word_count = len(words)
    
    # Exact matches for very short messages
    exact_matches = {
        "hi", "hello", "hey", "yo", "sup", "wassup", "hola",
        "salam", "assalam", "aoa", "slm", "salam alaikum",
        "thanks", "thank you", "shukriya", "thankyou", "thx",
        "bye", "goodbye", "ok bye", "alright bye", "see you",
        "ok", "okay", "hmm", "alright", "sure", "fine", "cool", "nice",
        "no", "nahi", "nope", "nothing", "no thanks",
        "yes", "yeah", "yep", "haan", "ji",
        "good morning", "good afternoon", "good evening", "good night",
    }
    if msg_lower in exact_matches:
        return True
    
    # Short messages (1-4 words) that match casual patterns
    if word_count <= 4:
        casual_words = {
            "hi", "hello", "hey", "yo", "sup", "wassup", "hola",
            "salam", "assalam", "aoa", "slm",
            "thanks", "thankyou", "shukriya", "thx",
            "bye", "goodbye",
            "ok", "okay", "hmm", "alright", "sure", "fine", "cool", "nice",
            "no", "nahi", "nope", "nothing",
            "yes", "yeah", "yep", "haan", "ji",
        }
        if any(w in casual_words for w in words):
            # But NOT if it also contains an HR keyword — those should go through retrieval
            hr_signals = {"leave", "salary", "pay", "benefit", "timing", "policy", "loan",
                          "advance", "probation", "notice", "resign", "sick", "casual",
                          "annual", "maternity", "paternity", "insurance", "attendance",
                          "holiday", "shift", "overtime", "increment", "vehicle", "ticket",
                          "chutti", "tankha", "naukri", "cutti"}
            if not any(w in hr_signals for w in words):
                return True
    
    # Rude/dismissive patterns (short messages with rude words)
    if word_count <= 5:
        rude_patterns = {"get lost", "shut up", "useless", "stupid", "go away", "leave me", 
                         "waste", "so what", "who cares", "whatever", "dont care"}
        if msg_lower in rude_patterns or any(p in msg_lower for p in rude_patterns):
            return True
    
    return False



# ── Retrieval Thresholds — Tuned for precision with GPT-4o-mini ──────────────
# RRF scores are small (e.g. 0.016–0.033), so threshold must be calibrated
RELEVANCE_THRESHOLD = 0.008   # Strict enough to block noise, loose enough for valid hits
# Cross-encoder logit > 0 means > 50% relevance probability
RERANK_THRESHOLD = 0.0
# If ALL chunks fail rerank threshold, fall back to this many top chunks
RERANK_FALLBACK_N = 2


def _deduplicate_chunks(chunks: List[Dict[str, str]], similarity_threshold: float = 0.85) -> List[Dict[str, str]]:
    """Remove near-duplicate chunks based on word overlap ratio.
    Prevents the LLM from seeing the same information repeated across overlapping windows."""
    if len(chunks) <= 1:
        return chunks

    unique = [chunks[0]]
    for candidate in chunks[1:]:
        candidate_words = set(candidate["text"].lower().split())
        is_duplicate = False
        for existing in unique:
            existing_words = set(existing["text"].lower().split())
            if not candidate_words or not existing_words:
                continue
            overlap = len(candidate_words & existing_words) / min(len(candidate_words), len(existing_words))
            if overlap >= similarity_threshold:
                is_duplicate = True
                break
        if not is_duplicate:
            unique.append(candidate)
    return unique


class RAGPipeline:
    def __init__(
        self,
        vector_store: FaissVectorStore,
        llm_service: LLMService,
        reranker: RerankerService,
        top_k: int,
        max_context_chunks: int
    ) -> None:
        self.vector_store = vector_store
        self.llm_service = llm_service
        self.reranker = reranker
        self.top_k = top_k
        self.max_context_chunks = max_context_chunks

    async def chat(self, message: str, history: List[Dict[str, str]], user_name: str = None) -> Dict[str, object]:
        # ── Cache check: return instantly for repeated identical questions ──
        key = _cache_key(message)
        if key in _answer_cache:
            _answer_cache.move_to_end(key)  # Mark as recently used
            logger.info("Cache HIT for: '%s'", message[:40])
            return _answer_cache[key]

        # ── Step 0: Resolve follow-up pronouns for RETRIEVAL only ──
        # "how to get them" (after "sick leaves") → "how to get sick leaves"
        # The LLM still gets the original message + history for natural conversation
        retrieval_query = _resolve_followup(message, history)

        # ── Step 0.5: Greetings / Casual — skip retrieval entirely ──
        # LLM handles greetings, casual chat, rude messages through conversational intelligence
        if _is_greeting(message):
            logger.info("Greeting/casual detected — skipping retrieval: '%s'", message[:40])
            reply = await self.llm_service.answer(
                question=message,
                chunks=[],
                history=history,
                user_name=user_name
            )
            result = {"reply": reply, "retrieved_chunks": []}
            # Don't cache greetings — they should feel natural, not repeated
            return result

        # ── Step 1: Intelligent Rewrite + Local Expansion ──
        # First, use LLM to understand intent perfectly and rewrite
        rewritten_query = await self.llm_service.rewrite_query(retrieval_query, history)
        logger.info("Intelligent Rewrite: '%s' → '%s'", message[:40], rewritten_query)

        # ── Step 1.5: Intercept special rewriter signals ──
        # NON_ENGLISH_QUERY  → user wrote in a non-English language; ask them to use English
        # NON_POLICY_PEOPLE_QUERY → user asked about a person/org detail not in the handbook
        signal = rewritten_query.strip()
        if signal in ("NON_ENGLISH_QUERY", "NON_POLICY_PEOPLE_QUERY"):
            # Safety check: if the original message matches a known HR keyword,
            # override the signal and proceed — the rewriter was wrong.
            msg_lower = message.lower().strip()
            known_keyword = any(kw in msg_lower for kw, _ in _INTENT_MAP)
            if known_keyword and signal == "NON_ENGLISH_QUERY":
                logger.info(
                    "NON_ENGLISH override: '%s' matches known keyword — proceeding with retrieval",
                    message[:40]
                )
                rewritten_query = message  # Use original message for retrieval
            else:
                logger.info("Special signal '%s' — skipping retrieval for: '%s'", signal, message[:40])
                reply = await self.llm_service.answer(
                    question=message,
                    chunks=[],
                    history=history,
                    user_name=user_name
                )
                return {"reply": reply, "retrieved_chunks": []}

        # Then, expand that rewritten query with local rules for maximum coverage
        queries = _expand_query_locally(rewritten_query)
        
        # ALSO expand the ORIGINAL message through the intent map.
        # This is critical: the rewriter is non-deterministic and may mangle
        # known HR terms (e.g. "prorata" → "proportional calculation"), causing
        # the intent map to miss on the rewrite. By expanding the original too,
        # we guarantee that known keywords always trigger correct retrieval.
        if message.lower().strip() != rewritten_query.lower().strip():
            original_expansions = _expand_query_locally(message)
            for oq in original_expansions:
                if oq not in queries:
                    queries.append(oq)
        
        # Ensure the original message is also included if not already there
        if message not in queries:
            queries.insert(1, message)
        
        logger.info("Final Retrieval Queries (%d): %s", len(queries), queries)

        # ── Step 2: Batched hybrid search ──
        all_retrieved = self.vector_store.multi_search(queries, top_k=self.top_k)

        # ── Step 3: Initial relevance filter ──
        initial_chunks = [c for c in all_retrieved if c["score"] >= RELEVANCE_THRESHOLD]

        if not initial_chunks:
            logger.info("No relevant chunks found for: '%s' — returning no-info response", message)
            # Pass empty chunks; LLM is instructed to say "I don't have that information"
            reply = await self.llm_service.answer(
                question=message,
                chunks=[],
                history=history,
                user_name=user_name
            )
            return {"reply": reply, "retrieved_chunks": []}

        # ── Step 4: Deep reranking via Cross-Encoder ──
        # Enrich the reranker query with the first expanded variant
        rerank_query = retrieval_query
        if len(queries) > 1:
            rerank_query = f"{retrieval_query} {queries[1]}"

        reranked_chunks = self.reranker.rerank(rerank_query, initial_chunks, top_n=self.max_context_chunks)

        # Filter by rerank score threshold
        final_chunks = [c for c in reranked_chunks if c.get("rerank_score", 0) > RERANK_THRESHOLD]

        # ── Smart Fallback: if ALL chunks fail the threshold, use the top N anyway ──
        # This prevents the bot from saying "I don't have info" when content WAS retrieved
        if not final_chunks and reranked_chunks:
            final_chunks = reranked_chunks[:RERANK_FALLBACK_N]
            logger.info(
                "Rerank fallback activated — all chunks below threshold (best score=%.3f), using top %d",
                reranked_chunks[0].get("rerank_score", 0),
                len(final_chunks)
            )

        # ── Step 4.5: Deduplicate near-identical chunks ──
        final_chunks = _deduplicate_chunks(final_chunks)

        logger.info(
            "Pipeline: retrieved=%d → relevance_filtered=%d → reranked=%d → final=%d (best_score=%.3f)",
            len(all_retrieved), len(initial_chunks), len(reranked_chunks), len(final_chunks),
            reranked_chunks[0].get("rerank_score", 0) if reranked_chunks else 0.0
        )

        # ── Step 5: Generate answer with top-ranked context ──
        # NOTE: Original message (not resolved) is passed to LLM — it has history context
        reply = await self.llm_service.answer(
            question=message,
            chunks=final_chunks,
            history=history,
            user_name=user_name
        )
        result = {"reply": reply, "retrieved_chunks": final_chunks}

        # ── Populate cache (evict oldest if at capacity) ──
        _answer_cache[key] = result
        if len(_answer_cache) > _CACHE_MAX:
            _answer_cache.popitem(last=False)  # Remove least-recently-used
        logger.info("Cache MISS — stored answer for: '%s'", message[:40])

        return result