Update main.py

main.py

@@ -1,3 +1,4 @@
+
 import os
 import json
 import re
@@ -19,21 +20,25 @@ from services.kb_creation import (
     get_best_steps_section_text,
     get_best_errors_section_text,
 )
+
 from services.login import router as login_router
 from services.generate_ticket import get_valid_token, create_incident
 
 VERIFY_SSL = os.getenv("SERVICENOW_SSL_VERIFY", "true").lower() in ("1", "true", "yes")
 GEMINI_SSL_VERIFY = os.getenv("GEMINI_SSL_VERIFY", "true").lower() in ("1", "true", "yes")
 
+
 def safe_str(e: Any) -> str:
     try:
         return builtins.str(e)
     except Exception:
         return "<error stringify failed>"
 
+
 load_dotenv()
 os.environ["POSTHOG_DISABLED"] = "true"
 
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     try:
@@ -47,6 +52,7 @@ async def lifespan(app: FastAPI):
         print(f"[KB] ingestion failed: {safe_str(e)}")
     yield
 
+
 app = FastAPI(lifespan=lifespan)
 app.include_router(login_router)
 
@@ -59,24 +65,28 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-# ------------------ Models ------------------
+# ---------------------- Models ----------------------
 class ChatInput(BaseModel):
     user_message: str
     prev_status: Optional[str] = None
     last_issue: Optional[str] = None
 
+
 class IncidentInput(BaseModel):
     short_description: str
     description: str
     mark_resolved: Optional[bool] = False
 
+
 class TicketDescInput(BaseModel):
     issue: str
 
+
 class TicketStatusInput(BaseModel):
     sys_id: Optional[str] = None
     number: Optional[str] = None
 
+
 STATE_MAP = {
     "1": "New",
     "2": "In Progress",
@@ -92,8 +102,10 @@ GEMINI_URL = (
     f"gemini-2.5-flash-lite:generateContent?key={GEMINI_API_KEY}"
 )
 
-# ------------------ Helpers ------------------
+# ---------------------- Helpers ----------------------
 NUMBERING_STYLE = os.getenv("NUMBERING_STYLE", "digit").lower()  # 'digit' or 'step'
+
+
 def _normalize_lines(text: str) -> List[str]:
     raw = (text or "")
     try:
@@ -102,44 +114,34 @@ def _normalize_lines(text: str) -> List[str]:
         return [raw.strip()] if raw.strip() else []
 
 
-def _ensure_numbering(text: str) -> str:
-    """
-    Render numbered steps robustly:
-    - Handles single-paragraph SOP content that already contains inline markers.
-    - Strips ANY leading marker per segment (classic, bullets, and circled numerals).
-    - Uses circled numerals (①..⑳) so the UI won't treat them as ordered lists.
-    - Falls back to 'N)' beyond 20 steps.
-    """
+def _normalize_for_match(text: str) -> str:
+    t = (text or "").lower()
+    t = re.sub(r"[^\w\s]", " ", t)
+    t = re.sub(r"\s+", " ", t).strip()
+    return t
+
 
+def _ensure_numbering(text: str) -> str:
     # 0) Strip hidden chars that can break regexes
     text = re.sub(r"[\u2060\u200B]", "", text or "")
 
-    # --- Helpers ---
     def strip_prefix_any(s: str) -> str:
-        """
-        Remove *any* leading list marker:
-          - classic: '1)', '2.', 'Step 3:', bullets '- * •'
-          - circled numerals: ①..⑳ (U+2460..U+2473)
-          - optional hidden chars before them
-        """
         return re.sub(
-            r"^\s*(?:[\u2060\u200B]*"                # optional zero-width prefix
-            r"(?:\d+\s*[.)]"                         # '1)' '2.' '3 )'
-            r"|(?:step\s*\d+:?)"                     # 'Step 3:' (case-insensitive handled below)
-            r"|[\-\*\u2022]"                         # bullets '-', '*', '•'
-            r"|[\u2460-\u2473]"                      # circled numerals ①..⑳
-            r"))\s*",
+            r"^\s*(?:[\u2060\u200B]*"
+            r"(?:\d+\s*[.)])"
+            r"|(?:step\s*\d+:?)"
+            r"|(?:[\-\*\u2022])"
+            r"|(?:[\u2460-\u2473])"
+            r")\s*",
             "",
             (s or "").strip(),
-            flags=re.IGNORECASE
+            flags=re.IGNORECASE,
         )
 
-    # 1) Normalize lines
     lines = [ln.strip() for ln in (text or "").splitlines() if ln and ln.strip()]
     if not lines:
         return text or ""
 
-    # 2) Merge dangling numeric-only lines with following text
     merged: List[str] = []
     i = 0
     while i < len(lines):
@@ -155,55 +157,149 @@ def _ensure_numbering(text: str) -> str:
     if not para:
         return ""
 
-    # 3) Neutralize inline markers to a hard delimiter '|||'
-    #    - classic numbers: ' 2) ', ' 3. '
-    #    - circled numerals: ' ② ' etc.
-    para_clean = re.sub(r"(?:(?<=^)|(?<=\s))\d+[.)]\s+", "|||", para)
-    para_clean = re.sub(r"(?:(?<=^)|(?<=\s))[\u2460-\u2473]\s+", "|||", para_clean)
-    para_clean = re.sub(r"(?i)\bstep\s*\d+\s*:\s*", "|||", para_clean)
+    para_clean = re.sub(r"(?:^|\s)\d+[.)]\s+", "\n\n\n", para)
+    para_clean = re.sub(r"(?:^|\s)[\u2460-\u2473]\s+", "\n\n\n", para_clean)
+    para_clean = re.sub(r"(?i)\bstep\s*\d+\s*:\s*", "\n\n\n", para_clean)
 
-    # 4) Split → segments
-    segments = [seg.strip() for seg in para_clean.split("|||") if seg.strip()]
+    segments = [seg.strip() for seg in para_clean.split("\n\n\n") if seg.strip()]
     if len(segments) < 2:
         tmp = [ln.strip() for ln in para.splitlines() if ln.strip()]
-        segments = tmp if len(tmp) > 1 else [seg.strip() for seg in re.split(r"(?<=[.!?])\s+|;\s+", para) if seg.strip()]
+        segments = tmp if len(tmp) > 1 else [seg.strip() for seg in re.split(r"(?<=[.!?])\s+|\s*;\s+", para) if seg.strip()]
 
-    # 5) Strip ANY leading marker from each segment (including circled ones)
     segments = [strip_prefix_any(seg) for seg in segments if seg.strip()]
 
-    # 6) Numbering: circled numerals for 1..20; then 'N)' style
     circled = {
-        1:"①",2:"②",3:"③",4:"④",5:"⑤",6:"⑥",7:"⑦",8:"⑧",9:"⑨",10:"⑩",
-        11:"⑪",12:"⑫",13:"⑬",14:"⑭",15:"⑮",16:"⑯",17:"⑰",18:"⑱",19:"⑲",20:"⑳",
+        1:"\u2460",2:"\u2461",3:"\u2462",4:"\u2463",5:"\u2464",6:"\u2465",7:"\u2466",8:"\u2467",9:"\u2468",10:"\u2469",
+        11:"\u246a",12:"\u246b",13:"\u246c",14:"\u246d",15:"\u246e",16:"\u246f",17:"\u2470",18:"\u2471",19:"\u2472",20:"\u2473",
     }
     out: List[str] = []
     for idx, seg in enumerate(segments, start=1):
         marker = circled.get(idx, f"{idx})")
         out.append(f"{marker} {seg}")
-
     return "\n".join(out)
 
 
+STRICT_OVERLAP = 3
+MAX_SENTENCES_STRICT = 4
+MAX_SENTENCES_CONCISE = 3
+
+PERM_QUERY_TERMS = [
+    "permission", "permissions", "access", "access right", "authorization", "authorisation",
+    "role", "role access", "security", "security profile", "privilege", "not allowed", "not authorized", "denied",
+]
+PERM_SYNONYMS = (
+    "permission", "permissions", "access", "authorization", "authorisation",
+    "role", "role mapping", "security profile", "not allowed", "not authorized", "denied", "insufficient"
+)
+
+NAV_LINE_REGEX = re.compile(r"(navigate\s+to|login|log in|menu|screen)", re.IGNORECASE)
+ERROR_STARTS = (
+    "error", "resolution", "fix", "verify", "check",
+    "permission", "access", "authorization", "authorisation",
+    "role", "role mapping", "security profile", "escalation", "not allowed", "not authorized", "denied"
+)
+
+# ---------------------- NEW: Generic dedupe helpers ----------------------
+def _canonical_line_key(s: str) -> str:
+    """Normalize a line into a key used for deduplication."""
+    t = _normalize_for_match(s)  # lower + punctuation stripped
+    t = re.sub(r'^\s*[\-\*\u2022]\s*', '', t)          # bullets
+    t = re.sub(r'^\s*\d+\s*[.)]\s*', '', t)            # "1)" or "2."
+    t = re.sub(r'^\s*[\u2460-\u2473]\s*', '', t)       # circled numerals
+    return re.sub(r'\s+', ' ', t).strip()
+
+
+def _dedupe_preserve_order(lines: List[str]) -> List[str]:
+    """Stable de-duplication by normalized content; preserves first occurrence order."""
+    seen = set()
+    out = []
+    for ln in lines:
+        key = _canonical_line_key(ln or "")
+        if key in seen:
+            continue
+        seen.add(key)
+        out.append(ln)
+    return out
 
-def _filter_error_lines_by_query(text: str, query: str, max_lines: int = 4) -> str:
-    """
-    Keep only error lines that contain key tokens from the user's query.
-    E.g., for 'putaway error', keep lines containing 'putaway' / 'error'.
-    """
-    q = _normalize_for_match(query)  # lower + punctuation stripped
-    q_terms = [t for t in q.split() if len(t) > 2]
+
+def _dedupe_text_block(text: str) -> str:
+    """Split -> dedupe -> rejoin without changing wording."""
+    lines = _normalize_lines(text)
+    uniq = _dedupe_preserve_order(lines)
+    return "\n".join(uniq).strip()
+
+
+# ---------------------- Filters ----------------------
+def _split_sentences(ctx: str) -> List[str]:
+    raw_sents = re.split(r"(?<=[.!?])\s+|\n+|\-\s*\n|\*\s*", ctx or "")
+    return [s.strip() for s in raw_sents if s and len(s.strip()) > 2]
+
+
+def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str, Any]]:
+    ctx = (context or "").strip()
+    if not ctx or not query:
+        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
+    q_norm = _normalize_for_match(query)
+    q_terms = [t for t in q_norm.split() if len(t) > 2]
     if not q_terms:
-        return text or ""
+        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
 
+    sentences = _split_sentences(ctx)
+    matched_exact, matched_any = [], []
+    for s in sentences:
+        s_norm = _normalize_for_match(s)
+        is_bullet = bool(re.match(r"^[\-\*]\s*", s))
+        overlap = sum(1 for t in q_terms if t in s_norm) + (1 if is_bullet else 0)
+        if overlap >= STRICT_OVERLAP:
+            matched_exact.append(s)
+        elif overlap > 0:
+            matched_any.append(s)
+
+    if matched_exact:
+        kept = matched_exact[:MAX_SENTENCES_STRICT]
+        return "\n".join(_dedupe_preserve_order(kept)).strip(), {'mode': 'exact', 'matched_count': len(kept), 'all_sentences': len(sentences)}
+    if matched_any:
+        kept = matched_any[:MAX_SENTENCES_CONCISE]
+        return "\n".join(_dedupe_preserve_order(kept)).strip(), {'mode': 'concise', 'matched_count': len(kept), 'all_sentences': len(sentences)}
+
+    kept = sentences[:MAX_SENTENCES_CONCISE]
+    return "\n".join(_dedupe_preserve_order(kept)).strip(), {'mode': 'concise', 'matched_count': 0, 'all_sentences': len(sentences)}
+
+
+def _extract_navigation_only(text: str, max_lines: int = 6) -> str:
+    lines = _normalize_lines(text)
     kept: List[str] = []
-    for ln in _normalize_lines(text):
-        ln_norm = _normalize_for_match(ln)
-        if any(t in ln_norm for t in q_terms):
+    for ln in lines:
+        if NAV_LINE_REGEX.search(ln) or ln.lower().startswith("log in") or ln.lower().startswith("login"):
             kept.append(ln)
         if len(kept) >= max_lines:
             break
+    return "\n".join(_dedupe_preserve_order(kept)).strip() if kept else _dedupe_text_block(text or "")
+
+
+def _extract_errors_only(text: str, max_lines: int = 12) -> str:
+    lines = _normalize_lines(text)
+    kept: List[str] = []
+    for ln in lines:
+        low = ln.lower()
+        if low.startswith(ERROR_STARTS) or any(key in low for key in ERROR_STARTS):
+            kept.append(ln)
+        if len(kept) >= max_lines:
+            break
+    return "\n".join(_dedupe_preserve_order(kept)).strip() if kept else _dedupe_text_block(text or "")
+
+
+def _filter_permission_lines(text: str, max_lines: int = 6) -> str:
+    lines = _normalize_lines(text)
+    kept: List[str] = []
+    for ln in lines:
+        low = ln.lower()
+        if any(k in low for k in PERM_SYNONYMS):
+            kept.append(ln)
+        if len(kept) >= max_lines:
+            break
+    return "\n".join(_dedupe_preserve_order(kept)).strip() if kept else _dedupe_text_block(text or "")
 
-    return "\n".join(kept).strip() if kept else (text or "").strip()
 
 def _friendly_permission_reply(raw: str) -> str:
     line = (raw or "").strip()
@@ -216,6 +312,7 @@ def _friendly_permission_reply(raw: str) -> str:
         return f"It seems to be an access issue: {line}. Please check your role mapping or request access."
     return line
 
+
 def _detect_language_hint(msg: str) -> Optional[str]:
     if re.search(r"[\u0B80-\u0BFF]", msg or ""):
         return "Tamil"
@@ -223,12 +320,14 @@ def _detect_language_hint(msg: str) -> Optional[str]:
         return "Hindi"
     return None
 
+
 def _build_clarifying_message() -> str:
     return (
         "It seems the issue isn’t resolved yet. Would you like to share a few details so I can check further, "
         "or should I raise a ServiceNow ticket for you?"
     )
 
+
 def _build_tracking_descriptions(issue_text: str, resolved_text: str) -> Tuple[str, str]:
     issue = (issue_text or "").strip()
     resolved = (resolved_text or "").strip()
@@ -240,6 +339,7 @@ def _build_tracking_descriptions(issue_text: str, resolved_text: str) -> Tuple[s
     ).strip()
     return short_desc, long_desc
 
+
 def _is_incident_intent(msg_norm: str) -> bool:
     intent_phrases = [
         "create ticket", "create a ticket", "raise ticket", "raise a ticket", "open ticket", "open a ticket",
@@ -249,6 +349,7 @@ def _is_incident_intent(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in intent_phrases)
 
+
 def _parse_ticket_status_intent(msg_norm: str) -> Dict[str, Optional[str]]:
     status_keywords = ["status", "ticket status", "incident status", "check status", "check ticket status", "check incident status"]
     if not any(k in msg_norm for k in status_keywords):
@@ -262,6 +363,7 @@ def _parse_ticket_status_intent(msg_norm: str) -> Dict[str, Optional[str]]:
                 return {"number": val.upper() if val.lower().startswith("inc") else val}
     return {"number": None, "ask_number": True}
 
+
 def _is_resolution_ack_heuristic(msg_norm: str) -> bool:
     phrases = [
         "it is resolved", "resolved", "issue resolved", "problem resolved",
@@ -270,6 +372,7 @@ def _is_resolution_ack_heuristic(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in phrases)
 
+
 def _has_negation_resolved(msg_norm: str) -> bool:
     neg_phrases = [
         "not resolved", "issue not resolved", "still not working", "not working",
@@ -277,6 +380,7 @@ def _has_negation_resolved(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in neg_phrases)
 
+
 def _classify_resolution_llm(user_message: str) -> bool:
     if not GEMINI_API_KEY:
         return False
@@ -298,157 +402,14 @@ Message: {user_message}"""
     except Exception:
         return False
 
-# ------------------ Filters ------------------
-STRICT_OVERLAP = 3
-MAX_SENTENCES_STRICT = 4
-MAX_SENTENCES_CONCISE = 3
-PERM_QUERY_TERMS = [
-    "permission", "permissions", "access", "access right", "authorization", "authorisation",
-    "role", "role access", "security", "security profile", "privilege", "not allowed", "not authorized", "denied",
-]
-PERM_SYNONYMS = (
-    "permission", "permissions", "access", "authorization", "authorisation",
-    "role", "role mapping", "security profile", "not allowed", "not authorized", "denied", "insufficient"
-)
-
-def _normalize_for_match(text: str) -> str:
-    t = (text or "").lower()
-    t = re.sub(r"[^\w\s]", " ", t)
-    t = re.sub(r"\s+", " ", t).strip()
-    return t
-
-def _split_sentences(ctx: str) -> List[str]:
-    raw_sents = re.split(r"(?<=[.!?])\s+|\n+|\-\s*|\*\s*", ctx or "")
-    return [s.strip() for s in raw_sents if s and len(s.strip()) > 2]
-
-def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str, Any]]:
-    ctx = (context or "").strip()
-    if not ctx or not query:
-        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
-    q_norm = _normalize_for_match(query)
-    q_terms = [t for t in q_norm.split() if len(t) > 2]
-    if not q_terms:
-        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
-    sentences = _split_sentences(ctx)
-    matched_exact, matched_any = [], []
-    for s in sentences:
-        s_norm = _normalize_for_match(s)
-        is_bullet = bool(re.match(r"^[\-\*]\s*", s))
-        overlap = sum(1 for t in q_terms if t in s_norm) + (1 if is_bullet else 0)
-        if overlap >= STRICT_OVERLAP:
-            matched_exact.append(s)
-        elif overlap > 0:
-            matched_any.append(s)
-    if matched_exact:
-        kept = matched_exact[:MAX_SENTENCES_STRICT]
-        return "\n".join(kept).strip(), {'mode': 'exact', 'matched_count': len(kept), 'all_sentences': len(sentences)}
-    if matched_any:
-        kept = matched_any[:MAX_SENTENCES_CONCISE]
-        return "\n".join(kept).strip(), {'mode': 'concise', 'matched_count': len(kept), 'all_sentences': len(sentences)}
-    kept = sentences[:MAX_SENTENCES_CONCISE]
-    return "\n".join(kept).strip(), {'mode': 'concise', 'matched_count': 0, 'all_sentences': len(sentences)}
-
-NAV_LINE_REGEX = re.compile(r"(navigate\s+to|login|log in|menu|screen)", re.IGNORECASE)
-
-def _extract_navigation_only(text: str, max_lines: int = 6) -> str:
-    lines = _normalize_lines(text)
-    kept: List[str] = []
-    for ln in lines:
-        if NAV_LINE_REGEX.search(ln) or ln.lower().startswith("log in") or ln.lower().startswith("login"):
-            kept.append(ln)
-        if len(kept) >= max_lines:
-            break
-    return "\n".join(kept).strip() if kept else (text or "").strip()
-
-ERROR_STARTS = (
-    "error", "resolution", "fix", "verify", "check",
-    "permission", "access", "authorization", "authorisation",
-    "role", "role mapping", "security profile", "escalation", "not allowed", "not authorized", "denied"
-)
-
-def _extract_errors_only(text: str, max_lines: int = 12) -> str:
-    lines = _normalize_lines(text)
-    kept: List[str] = []
-    for ln in lines:
-        low = ln.lower()
-        if low.startswith(ERROR_STARTS) or any(key in low for key in ERROR_STARTS):
-            kept.append(ln)
-        if len(kept) >= max_lines:
-            break
-    return "\n".join(kept).strip() if kept else (text or "").strip()
-
-def _filter_permission_lines(text: str, max_lines: int = 6) -> str:
-    lines = _normalize_lines(text)
-    kept: List[str] = []
-    for ln in lines:
-        low = ln.lower()
-        if any(k in low for k in PERM_SYNONYMS):
-            kept.append(ln)
-        if len(kept) >= max_lines:
-            break
-    return "\n".join(kept).strip() if kept else (text or "").strip()
-
-def _extract_escalation_line(text: str) -> Optional[str]:
-    """
-    Extract escalation path from SOP text and return a friendly sentence.
-    Prefers the line after 'Escalation Path:' or any arrow chain (-> or →).
-    """
-    if not text:
-        return None
-
-    lines = _normalize_lines(text)
-    if not lines:
-        return None
-
-    # Locate an escalation heading
-    start_idx = None
-    for i, ln in enumerate(lines):
-        low = ln.lower()
-        if "escalation" in low or "escalation path" in low or "escalate" in low:
-            start_idx = i
-            break
-
-    block = []
-    if start_idx is not None:
-        for j in range(start_idx, min(len(lines), start_idx + 6)):
-            if not lines[j].strip():
-                break
-            block.append(lines[j].strip())
-    else:
-        # Look for any arrows if heading not found
-        block = [ln.strip() for ln in lines if ("->" in ln or "→" in ln)]
-
-    if not block:
-        return None
-
-    text_block = " ".join(block)
-
-    m = re.search(r"escalation[^:]*:\s*(.+)", text_block, flags=re.IGNORECASE)
-    path = m.group(1).strip() if m else None
-
-    if not path:
-        arrow_lines = [ln for ln in block if ("->" in ln or "→" in ln)]
-        if arrow_lines:
-            path = arrow_lines[0]
-
-    if not path:
-        m2 = re.search(r"(operator.*?administrator|operator.*)", text_block, flags=re.IGNORECASE)
-        path = m2.group(1).strip() if m2 else None
-
-    if not path:
-        return None
 
-    path = path.replace("->", "→").strip()
-    path = re.sub(r"^(?i:escalation\s*path)\s*:\s*", "", path).strip()
-
-    return f"If you want to escalate the issue, follow: {path}"
-
-# ------------------ Health ------------------
+# ---------------------- Health ----------------------
 @app.get("/")
 async def health_check():
     return {"status": "ok"}
 
-# ------------------ Chat ------------------
+
+# ---------------------- Chat ----------------------
 @app.post("/chat")
 async def chat_with_ai(input_data: ChatInput):
     try:
@@ -463,6 +424,7 @@ async def chat_with_ai(input_data: ChatInput):
                 "options": [],
                 "debug": {"intent": "continue_conversation"},
             }
+
         if msg_norm in ("no", "no thanks", "nope"):
             return {
                 "bot_response": "No problem. Do you need assistance with any other issue?",
@@ -477,6 +439,7 @@ async def chat_with_ai(input_data: ChatInput):
         is_llm_resolved = _classify_resolution_llm(input_data.user_message)
         if _has_negation_resolved(msg_norm):
             is_llm_resolved = False
+
         if (not _has_negation_resolved(msg_norm)) and (_is_resolution_ack_heuristic(msg_norm) or is_llm_resolved):
             try:
                 short_desc, long_desc = _build_tracking_descriptions(input_data.last_issue, input_data.user_message)
@@ -636,12 +599,21 @@ async def chat_with_ai(input_data: ChatInput):
             if comb is not None: m["combined"] = comb
             items.append({"text": text, "meta": m})
 
+        # --- MERGE top chunks and DEDUPE before filtering ---
         selected = items[:max(1, 2)]
-        context_raw = "\n\n---\n\n".join([s["text"] for s in selected]) if selected else ""
+        if selected:
+            merged_lines = []
+            for s in selected:
+                merged_lines.extend(_normalize_lines(s["text"]))
+            merged_lines = _dedupe_preserve_order(merged_lines)
+            context_raw = "\n".join(merged_lines).strip()
+        else:
+            context_raw = ""
+
         filtered_text, filt_info = _filter_context_for_query(context_raw, input_data.user_message)
-        context = filtered_text
-        context_found = bool(context.strip())
+        context = _dedupe_text_block(filtered_text)  # <-- NEW: dedupe after filter
 
+        context_found = bool(context.strip())
         best_distance = min([d for d in distances if d is not None], default=None) if distances else None
         best_combined = max([c for c in combined if c is not None], default=None) if combined else None
         detected_intent = kb_results.get("user_intent", "neutral")
@@ -662,28 +634,26 @@ async def chat_with_ai(input_data: ChatInput):
                     sec = (top_meta or {}).get("section")
                     if sec: full_steps = get_section_text(best_doc, sec)
                 if full_steps:
-                    context = _ensure_numbering(full_steps)
+                    context = _ensure_numbering(full_steps)  # deterministic steps
+                    context = _dedupe_text_block(context)    # in case section repeats bullets
+
             elif detected_intent == "errors":
                 full_errors = get_best_errors_section_text(best_doc)
                 if full_errors:
-                    # Extract the error lines from the SOP
                     ctx_err = _extract_errors_only(full_errors, max_lines=12)
-
-                    # If the query mentions permissions, reduce to permission-only lines
                     if is_perm_query:
                         context = _filter_permission_lines(ctx_err, max_lines=6)
                     else:
-                        # Otherwise, keep only the lines relevant to the user's error words
-                        context = _filter_error_lines_by_query(ctx_err, input_data.user_message, max_lines=6)
-
-                    # Try to pick escalation path from the same SOP
+                        context = _filter_error_lines_by_query(ctx_err=input_data.user_message, text=ctx_err, max_lines=6) if False else _filter_error_lines_by_query(ctx_err, input_data.user_message, max_lines=6)  # keep signature
                     escalation_line = _extract_escalation_line(full_errors)
+                    context = _dedupe_text_block(context)
 
+                # If only one permission line remains, keep friendly line logic
                 lines = _normalize_lines(context)
                 if len(lines) == 1:
                     context = _friendly_permission_reply(lines[0])
 
-        # No context for errors → clarifying + ticket option
+        # No context for errors -> clarifying + ticket option
         if detected_intent == "errors" and not (context or "").strip():
             return {
                 "bot_response": _build_clarifying_message(),
@@ -698,7 +668,7 @@ async def chat_with_ai(input_data: ChatInput):
                 "debug": {"intent": "errors_no_context"},
             }
 
-        # Build LLM prompt
+        # LLM paraphrase for errors only (unchanged behavior)
         language_hint = _detect_language_hint(input_data.user_message)
         lang_line = f"Respond in {language_hint}." if language_hint else "Respond in a clear, polite tone."
         use_gemini = (detected_intent == "errors")
@@ -714,9 +684,9 @@ async def chat_with_ai(input_data: ChatInput):
 Return ONLY the rewritten guidance."""
         headers = {"Content-Type": "application/json"}
         payload = {"contents": [{"parts": [{"text": enhanced_prompt}]}]}
-
         bot_text = ""
         http_code = 0
+
         if use_gemini:
             try:
                 resp = requests.post(GEMINI_URL, headers=headers, json=payload, timeout=25, verify=GEMINI_SSL_VERIFY)
@@ -735,26 +705,22 @@ Return ONLY the rewritten guidance."""
         # Deterministic local formatting
         if detected_intent == "steps":
             bot_text = _ensure_numbering(context)
+            bot_text = _dedupe_text_block(bot_text)
         elif detected_intent == "errors":
             if not bot_text.strip() or http_code == 429:
-                # Fallback to SOP-derived filtered context (no generic)
                 bot_text = context.strip()
-
-            # Append SOP-specific escalation if available; otherwise NO escalation
             if escalation_line:
                 bot_text = (bot_text or "").rstrip() + "\n\n" + escalation_line
-        else:
-            bot_text = context
+            bot_text = _dedupe_text_block(bot_text)
 
-        # Status compute
         short_query = len((input_data.user_message or "").split()) <= 4
         gate_combined_ok = 0.60 if short_query else 0.55
         status = "OK" if (best_combined is not None and best_combined >= gate_combined_ok) else "PARTIAL"
         lower = (bot_text or "").lower()
         if ("partial" in lower) or ("may be partial" in lower) or ("closest" in lower) or ("may not fully" in lower):
             status = "PARTIAL"
-
         options = [{"type":"yesno","title":"Share details or raise a ticket?"}] if status == "PARTIAL" else []
+
         return {
             "bot_response": bot_text,
             "status": status,
@@ -776,95 +742,57 @@ Return ONLY the rewritten guidance."""
                 "best_doc": best_doc,
             },
         }
-
     except HTTPException:
         raise
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
-def _set_incident_resolved(sys_id: str) -> bool:
-    try:
-        token = get_valid_token()
-        instance_url = os.getenv("SERVICENOW_INSTANCE_URL")
-        if not instance_url:
-            print("[SN PATCH resolve] missing SERVICENOW_INSTANCE_URL")
-            return False
-        headers = {
-            "Authorization": f"Bearer {token}",
-            "Accept": "application/json",
-            "Content-Type": "application/json",
-        }
-        url = f"{instance_url}/api/now/table/incident/{sys_id}"
 
-        close_code_val = os.getenv("SERVICENOW_CLOSE_CODE", "Solution provided")
-        close_notes_val = os.getenv("SERVICENOW_RESOLUTION_NOTES", "Issue resolved, user confirmed")
-        caller_sysid = os.getenv("SERVICENOW_CALLER_SYSID")
-        resolved_by_sysid = os.getenv("SERVICENOW_RESOLVED_BY_SYSID")
-        assign_group = os.getenv("SERVICENOW_ASSIGNMENT_GROUP_SYSID")
-        require_progress = os.getenv("SERVICENOW_REQUIRE_IN_PROGRESS_FIRST", "false").lower() in ("1", "true", "yes")
+def _extract_escalation_line(text: str) -> Optional[str]:
+    if not text:
+        return None
+    lines = _normalize_lines(text)
+    if not lines:
+        return None
 
-        if require_progress:
-            try:
-                resp1 = requests.patch(url, headers=headers, json={"state": "2"}, verify=VERIFY_SSL, timeout=25)
-                print(f"[SN PATCH progress] status={resp1.status_code} body={resp1.text[:500]}")
-            except Exception as e:
-                print(f"[SN PATCH progress] exception={safe_str(e)}")
+    start_idx = None
+    for i, ln in enumerate(lines):
+        low = ln.lower()
+        if "escalation" in low or "escalation path" in low or "escalate" in low:
+            start_idx = i
+            break
 
-        def clean(d: dict) -> dict:
-            return {k: v for k, v in d.items() if v is not None}
+    block = []
+    if start_idx is not None:
+        for j in range(start_idx, min(len(lines), start_idx + 6)):
+            if not lines[j].strip():
+                break
+            block.append(lines[j].strip())
+    else:
+        block = [ln.strip() for ln in lines if ("->" in ln or "→" in ln)]
 
-        payload_A = clean({
-            "state": "6",
-            "close_code": close_code_val,
-            "close_notes": close_notes_val,
-            "caller_id": caller_sysid,
-            "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
-            "work_notes": "Auto-resolve set by NOVA.",
-            "resolved_by": resolved_by_sysid,
-            "assignment_group": assign_group,
-        })
-        respA = requests.patch(url, headers=headers, json=payload_A, verify=VERIFY_SSL, timeout=25)
-        if respA.status_code in (200, 204):
-            return True
-        print(f"[SN PATCH resolve A] status={respA.status_code} body={respA.text[:500]}")
+    if not block:
+        return None
 
-        payload_B = clean({
-            "state": "Resolved",
-            "close_code": close_code_val,
-            "close_notes": close_notes_val,
-            "caller_id": caller_sysid,
-            "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
-            "work_notes": "Auto-resolve set by NOVA.",
-            "resolved_by": resolved_by_sysid,
-            "assignment_group": assign_group,
-        })
-        respB = requests.patch(url, headers=headers, json=payload_B, verify=VERIFY_SSL, timeout=25)
-        if respB.status_code in (200, 204):
-            return True
-        print(f"[SN PATCH resolve B] status={respB.status_code} body={respB.text[:500]}")
+    text_block = " ".join(block)
+    m = re.search(r"escalation[^:]*:\s*(.+)", text_block, flags=re.IGNORECASE)
+    path = m.group(1).strip() if m else None
+    if not path:
+        arrow_lines = [ln for ln in block if ("->" in ln or "→" in ln)]
+        if arrow_lines:
+            path = arrow_lines[0]
+    if not path:
+        m2 = re.search(r"(operator.*?administrator|operator.*)", text_block, flags=re.IGNORECASE)
+        path = m2.group(1).strip() if m2 else None
+    if not path:
+        return None
+
+    path = path.replace("->", "→").strip()
+    path = re.sub(r"^(?i:escalation\s*path)\s*:\s*", "", path).strip()
+    return f"If you want to escalate the issue, follow: {path}"
 
-        code_field = os.getenv("SERVICENOW_RESOLUTION_CODE_FIELD", "close_code")
-        notes_field = os.getenv("SERVICENOW_RESOLUTION_NOTES_FIELD", "close_notes")
-        payload_C = clean({
-            "state": "6",
-            code_field: close_code_val,
-            notes_field: close_notes_val,
-            "caller_id": caller_sysid,
-            "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
-            "work_notes": "Auto-resolve set by NOVA.",
-            "resolved_by": resolved_by_sysid,
-            "assignment_group": assign_group,
-        })
-        respC = requests.patch(url, headers=headers, json=payload_C, verify=VERIFY_SSL, timeout=25)
-        if respC.status_code in (200, 204):
-            return True
-        print(f"[SN PATCH resolve C] status={respC.status_code} body={respC.text[:500]}")
-        return False
-    except Exception as e:
-        print(f"[SN PATCH resolve] exception={safe_str(e)}")
-        return False
 
-# ------------------ Incident ------------------
+# ---------------------- Incident ----------------------
 @app.post("/incident")
 async def raise_incident(input_data: IncidentInput):
     try:
@@ -889,7 +817,8 @@ async def raise_incident(input_data: IncidentInput):
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
-# ------------------ Ticket description generation ------------------
+
+# ---------------------- Ticket description generation ----------------------
 @app.post("/generate_ticket_desc")
 async def generate_ticket_desc_ep(input_data: TicketDescInput):
     try:
@@ -913,6 +842,7 @@ async def generate_ticket_desc_ep(input_data: TicketDescInput):
             text = data.get("candidates", [{}])[0].get("content", {}).get("parts", [{}])[0].get("text", "").strip()
         except Exception:
             return {"ShortDescription": "", "DetailedDescription": "", "error": "Gemini parsing failed"}
+
         if text.startswith("```"):
             lines = [ln for ln in text.splitlines() if not ln.strip().startswith("```")]
             text = "\n".join(lines).strip()
@@ -927,7 +857,8 @@ async def generate_ticket_desc_ep(input_data: TicketDescInput):
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
-# ------------------ Incident status ------------------
+
+# ---------------------- Incident status ----------------------
 @app.post("/incident_status")
 async def incident_status(input_data: TicketStatusInput):
     try:
@@ -955,7 +886,6 @@ async def incident_status(input_data: TicketStatusInput):
         state_label = STATE_MAP.get(state_code, state_code)
         short = result.get("short_description", "")
         number = result.get("number", input_data.number or "unknown")
-
         return {
             "bot_response": (
                 f"**Ticket:** {number} \n"
@@ -969,3 +899,86 @@ async def incident_status(input_data: TicketStatusInput):
         }
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
+
+
+def _set_incident_resolved(sys_id: str) -> bool:
+    try:
+        token = get_valid_token()
+        instance_url = os.getenv("SERVICENOW_INSTANCE_URL")
+        if not instance_url:
+            print("[SN PATCH resolve] missing SERVICENOW_INSTANCE_URL")
+            return False
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+        url = f"{instance_url}/api/now/table/incident/{sys_id}"
+
+        close_code_val = os.getenv("SERVICENOW_CLOSE_CODE", "Solution provided")
+        close_notes_val = os.getenv("SERVICENOW_RESOLUTION_NOTES", "Issue resolved, user confirmed")
+        caller_sysid = os.getenv("SERVICENOW_CALLER_SYSID")
+        resolved_by_sysid = os.getenv("SERVICENOW_RESOLVED_BY_SYSID")
+        assign_group = os.getenv("SERVICENOW_ASSIGNMENT_GROUP_SYSID")
+        require_progress = os.getenv("SERVICENOW_REQUIRE_IN_PROGRESS_FIRST", "false").lower() in ("1", "true", "yes")
+
+        if require_progress:
+            try:
+                resp1 = requests.patch(url, headers=headers, json={"state": "2"}, verify=VERIFY_SSL, timeout=25)
+                print(f"[SN PATCH progress] status={resp1.status_code} body={resp1.text[:500]}")
+            except Exception as e:
+                print(f"[SN PATCH progress] exception={safe_str(e)}")
+
+        def clean(d: dict) -> dict:
+            return {k: v for k, v in d.items() if v is not None}
+
+        payload_A = clean({
+            "state": "6",
+            "close_code": close_code_val,
+            "close_notes": close_notes_val,
+            "caller_id": caller_sysid,
+            "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
+            "work_notes": "Auto-resolve set by NOVA.",
+            "resolved_by": resolved_by_sysid,
+            "assignment_group": assign_group,
+        })
+        respA = requests.patch(url, headers=headers, json=payload_A, verify=VERIFY_SSL, timeout=25)
+        if respA.status_code in (200, 204):
+            return True
+        print(f"[SN PATCH resolve A] status={respA.status_code} body={respA.text[:500]}")
+
+        payload_B = clean({
+            "state": "Resolved",
+            "close_code": close_code_val,
+            "close_notes": close_notes_val,
+            "caller_id": caller_sysid,
+            "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
+            "work_notes": "Auto-resolve set by NOVA.",
+            "resolved_by": resolved_by_sysid,
+            "assignment_group": assign_group,
+        })
+        respB = requests.patch(url, headers=headers, json=payload_B, verify=VERIFY_SSL, timeout=25)
+        if respB.status_code in (200, 204):
+            return True
+        print(f"[SN PATCH resolve B] status={respB.status_code} body={respB.text[:500]}")
+
+        code_field = os.getenv("SERVICENOW_RESOLUTION_CODE_FIELD", "close_code")
+        notes_field = os.getenv("SERVICENOW_RESOLUTION_NOTES_FIELD", "close_notes")
+        payload_C = clean({
+            "state": "6",
+            code_field: close_code_val,
+            notes_field: close_notes_val,
+            "caller_id": caller_sysid,
+            "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
+            "work_notes": "Auto-resolve set by NOVA.",
+            "resolved_by": resolved_by_sysid,
+            "assignment_group": assign_group,
+        })
+        respC = requests.patch(url, headers=headers, json=payload_C, verify=VERIFY_SSL, timeout=25)
+        if respC.status_code in (200, 204):
+            return True
+        print(f"[SN PATCH resolve C] status={respC.status_code} body={respC.text[:500]}")
+        return False
+    except Exception as e:
+        print(f"[SN PATCH resolve] exception={safe_str(e)}")
+        return False