Update main.py

main.py

@@ -12,7 +12,6 @@ from pydantic import BaseModel
 from dotenv import load_dotenv
 from datetime import datetime
 
-# ------------------------------ Imports ------------------------------
 # Import shared vocab from KB services
 from services.kb_creation import ACTION_SYNONYMS, MODULE_VOCAB
 from services.kb_creation import (
@@ -23,24 +22,25 @@ from services.kb_creation import (
     get_best_steps_section_text,
     get_best_errors_section_text,
     get_escalation_text,  # for escalation heading
-    bm25_docs,
 )
 from services.login import router as login_router
 from services.generate_ticket import get_valid_token, create_incident
 
-# ------------------------------ Config ------------------------------
 VERIFY_SSL = os.getenv("SERVICENOW_SSL_VERIFY", "true").lower() in ("1", "true", "yes")
 GEMINI_SSL_VERIFY = os.getenv("GEMINI_SSL_VERIFY", "true").lower() in ("1", "true", "yes")
 
+
 def safe_str(e: Any) -> str:
     try:
         return builtins.str(e)
     except Exception:
         return "<error stringify failed>"
 
+
 load_dotenv()
 os.environ["POSTHOG_DISABLED"] = "true"
 
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     try:
@@ -54,6 +54,7 @@ async def lifespan(app: FastAPI):
         print(f"[KB] ingestion failed: {safe_str(e)}")
     yield
 
+
 app = FastAPI(lifespan=lifespan)
 app.include_router(login_router)
 
@@ -72,18 +73,22 @@ class ChatInput(BaseModel):
     prev_status: Optional[str] = None
     last_issue: Optional[str] = None
 
+
 class IncidentInput(BaseModel):
     short_description: str
     description: str
     mark_resolved: Optional[bool] = False
 
+
 class TicketDescInput(BaseModel):
     issue: str
 
+
 class TicketStatusInput(BaseModel):
     sys_id: Optional[str] = None
     number: Optional[str] = None
 
+
 STATE_MAP = {
     "1": "New",
     "2": "In Progress",
@@ -102,6 +107,7 @@ GEMINI_URL = (
 # ------------------------------ Helpers ------------------------------
 NUMBERING_STYLE = os.getenv("NUMBERING_STYLE", "digit").lower()  # 'digit' or 'step'
 
+# Domain status terms (generic WMS domain words)
 DOMAIN_STATUS_TERMS = (
     "shipment", "order", "load", "trailer", "wave",
     "inventory", "putaway", "receiving", "appointment",
@@ -109,6 +115,7 @@ DOMAIN_STATUS_TERMS = (
     "asn", "grn", "pick", "picking"
 )
 
+# --- Generic error families (SOP-wide, reusable in gating and line selection) ---
 ERROR_FAMILY_SYNS = {
     "NOT_FOUND": (
         "not found", "missing", "does not exist", "doesn't exist",
@@ -137,10 +144,13 @@ ERROR_FAMILY_SYNS = {
     ),
 }
 
-# Runtime synonym extension
+# ----- local extension so runtime filtering is precise even without re-ingest -----
+# (Does NOT override your KB synonyms—just augments them at runtime.)
 ACTION_SYNONYMS_EXT: Dict[str, List[str]] = {}
 for k, v in ACTION_SYNONYMS.items():
-    ACTION_SYNONYMS_EXT[k] = list(v)
+    ACTION_SYNONYMS_EXT[k] = list(v)  # copy
+
+# Extend with SOP phrasing (appointments often say 'updation', 'deletion', 'reschedule')
 ACTION_SYNONYMS_EXT.setdefault("create", []).extend(["appointment creation", "create appointment"])
 ACTION_SYNONYMS_EXT.setdefault("update", []).extend([
     "updation", "reschedule", "change time", "change date", "change slot",
@@ -148,7 +158,9 @@ ACTION_SYNONYMS_EXT.setdefault("update", []).extend([
 ])
 ACTION_SYNONYMS_EXT.setdefault("delete", []).extend(["deletion", "delete appointment", "cancel appointment"])
 
+
 def _detect_error_families(msg: str) -> list:
+    """Return matching error family names found in the message (generic across SOPs)."""
     low = (msg or "").lower()
     low_norm = re.sub(r"[^\w\s]", " ", low)
     low_norm = re.sub(r"\s+", " ", low_norm).strip()
@@ -158,11 +170,21 @@ def _detect_error_families(msg: str) -> list:
             fams.append(fam)
     return fams
 
-# ------------------------------ Title/section global helpers ------------------------------
+
+# --- Action-targeted steps selector (uses existing KB metadata) ---
+from services.kb_creation import bm25_docs, get_section_text
+
 def _get_steps_for_action(best_doc: str, actions: list) -> Optional[str]:
+    """
+    Return the full text of the steps section whose action_tag matches the user's intent.
+    e.g., actions=['update'] -> section: "Appointment schedule updation"
+          actions=['delete'] -> section: "Appointment deletion"
+          actions=['create'] -> section: "Appointment Creation"
+    """
     if not best_doc or not actions:
         return None
     act_set = set(a.strip().lower() for a in actions if a)
+    # Collect candidate sections in this doc that are 'steps' and have an action_tag we need
     candidates = []
     for d in bm25_docs:
         m = d.get("meta", {})
@@ -170,40 +192,24 @@ def _get_steps_for_action(best_doc: str, actions: list) -> Optional[str]:
             tag = (m.get("action_tag") or "").strip().lower()
             if tag and tag in act_set:
                 candidates.append(m.get("section"))
+    # Prefer the first matched section with non-empty text
     for title in candidates:
         txt = get_section_text(best_doc, title)
         if txt and txt.strip():
             return txt.strip()
     return None
 
-def _get_steps_by_action_tag(best_doc: str, action: Optional[str]) -> Optional[str]:
-    """
-    Generic: return the section text from `best_doc` whose meta.action_tag == action.
-    No hard-coded SOP words; relies purely on KB metadata.
-    """
-    if not best_doc or not action:
-        return None
-    target = (action or "").strip().lower()
-    for d in bm25_docs:
-        m = d.get("meta", {}) or {}
-        if m.get("intent_tag") != "steps":
-            continue
-        if (m.get("filename") or "") != best_doc:
-            continue
-        tag = (m.get("action_tag") or "").strip().lower()
-        if tag and tag == target:
-            sec = (m.get("section") or "").strip()
-            if sec:
-                txt = get_section_text(best_doc, sec)
-                if txt and txt.strip():
-                    return txt.strip()
-    return None
 
 def _get_steps_for_action_global(actions: list, prefer_doc: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]:
+    """
+    Search ALL SOP docs for a 'steps' section whose action_tag matches one of `actions`.
+    Returns (doc_name, steps_text). Prefers sections from `prefer_doc` and the 'appointments' module.
+    """
     if not actions:
         return None, None
     act_set = set(a.strip().lower() for a in actions if a)
-    candidates: List[Tuple[float, str, str]] = []
+
+    candidates: List[Tuple[float, str, str]] = []  # (score, doc, text)
     for d in bm25_docs:
         m = d.get("meta", {}) or {}
         if m.get("intent_tag") != "steps":
@@ -222,38 +228,25 @@ def _get_steps_for_action_global(actions: list, prefer_doc: Optional[str] = None
             if "appointments" in mtags:
                 score += 0.3
             candidates.append((score, doc, txt.strip()))
+
     if not candidates:
         return None, None
+
     candidates.sort(key=lambda x: x[0], reverse=True)
     _, best_doc_global, best_text = candidates[0]
     return best_doc_global, best_text
 
+# --- Default section picker when query doesn't reveal action ---
 def _pick_default_action_section(best_doc: str) -> Optional[str]:
     """
-    Backward-compatible wrapper: prefer-action version with no preference.
-    """
-    return _pick_default_action_section_with_preference(best_doc, None)
-
-def _pick_default_action_section_with_preference(best_doc: str, prefer_action: Optional[str]) -> Optional[str]:
-    """
-    Generic: if there are sections tagged with the requested action (via action_tag),
-    prefer them; otherwise fall back to your original order.
+    If user actions are empty, prefer '...Creation' section,
+    else prefer '...Updation'/'...Update', else '...Deletion'/'...Cancel'.
+    Works generically for SOPs that use common headings.
     """
-    # 1) Prefer exact action_tag match
-    if prefer_action:
-        for d in bm25_docs:
-            m = d.get("meta", {}) or {}
-            if m.get("filename") == best_doc and m.get("intent_tag") == "steps":
-                tag = (m.get("action_tag") or "").strip().lower()
-                sec = (m.get("section") or "").strip()
-                if tag == prefer_action.lower() and sec:
-                    return sec
-
-    # 2) Fall back to your original generic order (no SOP-specific words)
     order = ("creation", "updation", "update", "deletion", "delete", "cancel")
     sections = []
     for d in bm25_docs:
-        m = d.get("meta", {}) or {}
+        m = d.get("meta", {})
         if m.get("filename") == best_doc and m.get("intent_tag") == "steps":
             title = (m.get("section") or "").strip().lower()
             if title:
@@ -264,146 +257,36 @@ def _pick_default_action_section_with_preference(best_doc: str, prefer_action: O
                 return t
     return sections[0] if sections else None
 
-# ------------------------------ Optional title-based fallback ------------------------------
-ACTION_SECTION_KEYS = {
-    "create": ("create", "creation", "appointment creation", "new appointment", "book", "schedule"),
-    "update": ("update", "updation", "reschedule", "change", "modify", "edit"),
-    "delete": ("delete", "deletion", "cancel", "remove", "void"),
-}
-
-def _pick_action_section(best_doc: str, action: Optional[str]) -> Optional[str]:
-    if not best_doc or not action:
-        return None
-    keys = ACTION_SECTION_KEYS.get(action.lower(), ())
-    if not keys:
-        return None
-    candidates = []
-    for d in bm25_docs:
-        m = d.get("meta", {}) or {}
-        if m.get("filename") == best_doc and (m.get("intent_tag") == "steps"):
-            sec = (m.get("section") or "").strip()
-            if not sec:
-                continue
-            low = sec.lower()
-            if any(k in low for k in keys):
-                score = 1.0
-                if action.lower() in low:
-                    score += 0.5
-                mtags = (m.get("module_tags") or "").lower()
-                if "appointments" in mtags:
-                    score += 0.2
-                candidates.append((score, sec))
-    if not candidates:
-        return None
-    candidates.sort(key=lambda x: x[0], reverse=True)
-    return candidates[0][1]
 
-def _get_steps_text_for_action(best_doc: str, action: Optional[str]) -> Optional[str]:
-    sec = _pick_action_section(best_doc, action)
-    if not sec:
-        return None
-    txt = get_section_text(best_doc, sec)
-    return (txt or "").strip() or None
-
-# ------------------------------ Save lines helpers ------------------------------
+# --- Harvest 'Save' lines from ALL steps chunks in the doc (generic across SOPs) ---
 SAVE_SYNS = ("save", "saving", "save changes", "click save", "press save", "select save")
 
-def _find_save_lines_in_section(section_text: str, max_lines: int = 2) -> str:
-    lines: List[str] = []
-    for ln in [x.strip() for x in (section_text or "").splitlines() if x.strip()]:
-        low = ln.lower()
-        if any(s in low for s in SAVE_SYNS):
-            lines.append(ln)
-            if len(lines) >= max_lines:
-                break
-    return "\n".join(lines)
-
-# ------------------------------ Generic boundary cutter (metadata + action-family) ------------------------------
-def _build_doc_section_index(best_doc: str) -> Dict[str, Optional[str]]:
+def _find_save_lines_in_doc(best_doc: str, max_lines: int = 2) -> str:
     """
-    Build a dictionary for the given doc:
-    { lower(section_title): lower(action_tag or None) }
+    Pulls up to max_lines lines that mention 'save' from any steps chunk in best_doc.
+    Returns a \n-joined string or empty if none found.
     """
-    index: Dict[str, Optional[str]] = {}
+    lines: List[str] = []
     for d in bm25_docs:
-        m = d.get("meta", {}) or {}
-        if m.get("filename") == best_doc and m.get("intent_tag") == "steps":
-            sec = (m.get("section") or "").strip()
-            tag = (m.get("action_tag") or "").strip().lower() or None
-            if sec:
-                index[sec.lower()] = tag
-    return index
-
-def _cut_at_next_boundary_generic(section_text: str, best_doc: str, current_action: Optional[str]) -> str:
-    """
-    Stop when we hit:
-      1) Any known section heading (from KB metadata) that belongs to a *different* action_tag, OR
-      2) Any heading-like OR numbered line that contains generic action keywords for a *different* action family.
-
-    Generic & future-proof: no SOP-specific terms.
-    """
-    if not (section_text or "").strip():
-        return section_text
-
-    index = _build_doc_section_index(best_doc)  # {lower(section_title): action_tag}
-    known_headings = set(index.keys())
-
-    ACTION_FAMILIES = {
-        "create": ("create", "creation", "new"),
-        "update": ("update", "updation", "reschedule", "edit", "modify", "change"),
-        "delete": ("delete", "deletion", "cancel", "remove", "void"),
-    }
-
-    STEP_PREFIX_RX = re.compile(r"^\s*(?:[\u2460-\u2473]|\d+\s*[.)]|[-*•])")
-
-    def detect_action_family_in_line(line_low: str) -> Optional[str]:
-        for fam, toks in ACTION_FAMILIES.items():
-            for t in toks:
-                if re.search(rf"\b{re.escape(t)}\b", line_low, flags=re.IGNORECASE):
-                    return fam
-        return None
-
-    def is_heading_like(raw_line: str, line_low: str) -> bool:
-        if (":" in raw_line) or ("–" in raw_line) or re.match(r"^\s*[-–]\s*", raw_line):
-            return True
-        if any(h in line_low for h in known_headings):
-            return True
-        if len(raw_line.strip()) <= 140:
-            words = re.findall(r"[A-Za-z][A-Za-z]+", raw_line)
-            cap_ratio = sum(1 for w in words if (w[0].isupper() or w.isupper())) / (len(words) or 1)
-            if cap_ratio >= 0.40:
-                return True
-        return False
-
-    lines = [ln for ln in (section_text or "").splitlines() if ln.strip()]
-    out: List[str] = []
-
-    for ln in lines:
-        low = ln.lower().strip()
-
-        # 1) Metadata heading boundary
-        matched_heading = None
-        for h in known_headings:
-            if h in low:
-                matched_heading = h
-                break
-
-        if matched_heading:
-            next_action = index.get(matched_heading)
-            if (current_action and next_action and next_action != current_action) or (current_action is None):
-                break
+        m = d.get("meta", {})
+        if m.get("filename") != best_doc or m.get("intent_tag") != "steps":
+            continue
+        t = (d.get("text") or "").strip()
+        for ln in [x.strip() for x in t.splitlines() if x.strip()]:
+            low = ln.lower()
+            if any(s in low for s in SAVE_SYNS):
+                lines.append(ln)
+                if len(lines) >= max_lines:
+                    return "\n".join(lines)
+    return "\n".join(lines)
 
-        # 2) Generic action boundary (works even if visible text != metadata title)
-        fam = detect_action_family_in_line(low)
-        if current_action and fam and fam != current_action:
-            if is_heading_like(ln, low) or STEP_PREFIX_RX.match(ln):
-                break
 
-        out.append(ln)
+def _is_domain_status_context(msg_norm: str) -> bool:
+    if "status locked" in msg_norm or "locked status" in msg_norm:
+        return True
+    return any(term in msg_norm for term in DOMAIN_STATUS_TERMS)
 
-    return "\n".join(out).strip()
 
-# ------------------------------ Text normalization / numbering ------------------------------
 def _normalize_lines(text: str) -> List[str]:
     raw = (text or "")
     try:
@@ -411,29 +294,41 @@ def _normalize_lines(text: str) -> List[str]:
     except Exception:
         return [raw.strip()] if raw.strip() else []
 
+
 def _ensure_numbering(text: str) -> str:
+    """
+    Normalize raw SOP steps into a clean numbered list using circled digits.
+    Robust against '1.', '1)', 'Step 1:', bullets ('-', '*', '•'), and circled digits.
+    """
     text = re.sub(r"[\u2060\u200B]", "", text or "")
     lines = [ln.strip() for ln in (text or "").splitlines() if ln and ln.strip()]
     if not lines:
         return text or ""
+
+    # Collapse lines into a block and then split on common step markers
     para = " ".join(lines).strip()
     if not para:
         return ""
-    para_clean = re.sub(r"(?:\b\d+[.)]\s+)", "\n\n\n", para)
-    para_clean = re.sub(r"(?:[\u2460-\u2473]\s+)", "\n\n\n", para_clean)
-    para_clean = re.sub(r"(?i)\bstep\s*\d+\s*:\s*", "\n\n\n", para_clean)
+
+    # Create hard breaks at typical step boundaries
+    para_clean = re.sub(r"(?:\b\d+[.)]\s+)", "\n\n\n", para)                 # 1. / 1)
+    para_clean = re.sub(r"(?:[\u2460-\u2473]\s+)", "\n\n\n", para_clean)     # circled digits
+    para_clean = re.sub(r"(?i)\bstep\s*\d+\s*:\s*", "\n\n\n", para_clean)    # Step 1:
     segments = [seg.strip() for seg in para_clean.split("\n\n\n") if seg.strip()]
+
+    # Fallback splitting if we didn't detect separators
     if len(segments) < 2:
         tmp = [ln.strip() for ln in para.splitlines() if ln.strip()]
         segments = tmp if len(tmp) > 1 else [seg.strip() for seg in re.split(r"(?<=[.!?])\s+|\s+;\s+", para) if seg.strip()]
 
+    # Strip any step prefixes
     def strip_prefix_any(s: str) -> str:
         return re.sub(
             r"^\s*(?:"
-            r"(?:\d+\s*[.)])"
-            r"|(?:step\s*\d+:?)"
-            r"|(?:[-*\u2022])"
-            r"|(?:[\u2460-\u2473])"
+            r"(?:\d+\s*[.)])"        # leading numbers 1., 2)
+            r"|(?:step\s*\d+:?)"     # Step 1:
+            r"|(?:[-*\u2022])"       # bullets
+            r"|(?:[\u2460-\u2473])"  # circled digits
             r")\s*",
             "",
             (s or "").strip(),
@@ -441,21 +336,33 @@ def _ensure_numbering(text: str) -> str:
         )
 
     clean_segments = [strip_prefix_any(seg) for seg in segments if seg.strip()]
-    circled = {i: chr(9311 + i) for i in range(1, 21)}  # ①..⑳
+    circled = {
+        1: "\u2460", 2: "\u2461", 3: "\u2462", 4: "\u2463", 5: "\u2464",
+        6: "\u2465", 7: "\u2466", 8: "\u2467", 9: "\u2468", 10: "\u2469",
+        11: "\u246a", 12: "\u246b", 13: "\u246c", 14: "\u246d", 15: "\u246e",
+        16: "\u246f", 17: "\u2470", 18: "\u2471", 19: "\u2472", 20: "\u2473"
+    }
     out = []
     for idx, seg in enumerate(clean_segments, start=1):
         marker = circled.get(idx, f"{idx})")
         out.append(f"{marker} {seg}")
     return "\n".join(out)
 
-# ------------------------------ Next-step helpers ------------------------------
+
+# --- Next-step helpers (generic; SOP-agnostic) ---
 def _norm_text(s: str) -> str:
     s = (s or "").lower()
     s = re.sub(r"[^\w\s]", " ", s)
     s = re.sub(r"\s+", " ", s).strip()
     return s
 
+
 def _split_sop_into_steps(numbered_text: str) -> list:
+    """
+    Split a numbered/bulleted SOP block (already passed through _ensure_numbering)
+    into atomic steps. Returns a list of raw step strings (order preserved).
+    Safe for circled digits, '1.' styles, and bullets.
+    """
     lines = [ln.strip() for ln in (numbered_text or "").splitlines() if ln.strip()]
     steps = []
     for ln in lines:
@@ -464,7 +371,9 @@ def _split_sop_into_steps(numbered_text: str) -> list:
             steps.append(cleaned)
     return steps
 
+
 def _soft_match_score(a: str, b: str) -> float:
+    # Simple Jaccard-like score on tokens for fuzzy matching
     ta = set(_norm_text(a).split())
     tb = set(_norm_text(b).split())
     if not ta or not tb:
@@ -473,132 +382,153 @@ def _soft_match_score(a: str, b: str) -> float:
     union = len(ta | tb)
     return inter / union if union else 0.0
 
+
 def _detect_next_intent(user_query: str) -> bool:
     q = _norm_text(user_query)
     keys = [
         "after", "after this", "what next", "whats next", "next step",
-        "then what", "following step", "continue", "subsequent", "proceed",
-        "next", "go next", "what is next", "what should be next"
+        "then what", "following step", "continue", "subsequent", "proceed"
     ]
     return any(k in q for k in keys)
 
-def _resolve_next_steps(user_query: str, numbered_text: str, max_next: int = 6, min_score: float = 0.30):
+
+def _resolve_next_steps(user_query: str, numbered_text: str, max_next: int = 6, min_score: float = 0.35):
     """
-    Smarter anchor: verb overlap boost + keyword anchors.
+    If 'what's next' intent is detected and we can reliably match the user's
+    referenced line to a SOP step, return ONLY the subsequent steps.
+    Else return None to preserve current behavior.
     """
     if not _detect_next_intent(user_query):
         return None
+
     steps = _split_sop_into_steps(numbered_text)
     if not steps:
         return None
 
-    q = _norm_text(user_query)
-    q_tokens = set(q.split())
-
-    anchors = [
-        ({"scan", "location"}, "scan location"),
-        ({"confirm", "item"}, "confirm item"),
-        ({"pick", "quantity"}, "pick quantity"),
-        ({"place", "picked"}, "place picked"),
-        ({"confirm", "completion"}, "confirm completion"),
-        ({"move", "pallet"}, "move pallet"),
-        ({"system", "prompts"}, "system prompts"),
-    ]
-
-    anchor_idx = -1
-    for need, phrase in anchors:
-        if need.issubset(q_tokens):
-            for i, s in enumerate(steps):
-                if phrase in _norm_text(s):
-                    anchor_idx = i
-                    break
-        if anchor_idx >= 0:
-            break
-
-    PROCEDURE_VERBS = {"scan", "confirm", "pick", "place", "move", "complete", "submit", "save"}
-    def verb_overlap_boost(step_text: str) -> float:
-        stoks = set(_norm_text(step_text).split())
-        overlap = len(PROCEDURE_VERBS & stoks & q_tokens)
-        return 0.12 * overlap
-
+    q = user_query or ""
     best_idx, best_score = -1, -1.0
-    if anchor_idx >= 0:
-        best_idx, best_score = anchor_idx, 1.0
-    else:
-        for idx, step in enumerate(steps):
-            base = _soft_match_score(user_query, step)
-            boost = verb_overlap_boost(step)
-            score = base + boost
-            if score > best_score:
-                best_score, best_idx = score, idx
+    for idx, step in enumerate(steps):
+        score = 1.0 if _norm_text(step) in _norm_text(q) else _soft_match_score(q, step)
+        if score > best_score:
+            best_score, best_idx = score, idx
 
     if best_idx < 0 or best_score < min_score:
-        return None
+        return None  # fallback to full SOP
+
     start = best_idx + 1
     if start >= len(steps):
-        return []
+        return []  # user is at final step
+
     end = min(start + max_next, len(steps))
     return steps[start:end]
 
+
 def _format_steps_as_numbered(steps: list) -> str:
-    circled = {i: chr(9311 + i) for i in range(1, 21)}  # ①..⑳
+    """Render a small list of steps with circled numbers for visual continuity."""
+    circled = {
+        1: "\u2460", 2: "\u2461", 3: "\u2462", 4: "\u2463", 5: "\u2464",
+        6: "\u2465", 7: "\u2466", 8: "\u2467", 9: "\u2468", 10: "\u2469",
+        11: "\u246a", 12: "\u246b", 13: "\u246c", 14: "\u246d", 15: "\u246e",
+        16: "\u246f", 17: "\u2470", 18: "\u2471", 19: "\u2472", 20: "\u2473"
+    }
     out = []
     for i, s in enumerate(steps, start=1):
         out.append(f"{circled.get(i, str(i))} {s}")
     return "\n".join(out)
 
-# ------------------------------ Context filter (ensure defined before /chat) ------------------------------
-def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str, Any]]:
+
+def _filter_error_lines_by_query(text: str, query: str, max_lines: int = 1) -> str:
     """
-    Keep only the most relevant sentences from the KB context for the query.
-    Returns (filtered_text, info_dict).
+    Pick the most relevant 'Common Errors & Resolution' bullet(s) for the user's message.
+    Generic (SOP-agnostic) scoring:
+      1) error-family match (NOT_FOUND/MISMATCH/LOCKED/PERMISSION/TIMEOUT/SYNC),
+      2) anchored starts (line begins with error heading),
+      3) multi-word overlap (bigrams/trigrams),
+      4) token overlap,
+      5) formatting bonus for bullets/headings.
+
+    Returns exactly `max_lines` best-scoring lines (defaults to 1).
     """
-    STRICT_OVERLAP = 3
-    MAX_SENTENCES_STRICT = 4
-    MAX_SENTENCES_CONCISE = 3
+    def _norm(s: str) -> str:
+        s = (s or "").lower()
+        s = re.sub(r"[^\w\s]", " ", s)
+        s = re.sub(r"\s+", " ", s).strip()
+        return s
+
+    def _ngrams(tokens: List[str], n: int) -> List[str]:
+        return [" ".join(tokens[i:i + n]) for i in range(len(tokens) - n + 1)]
+
+    def _families_for(s: str) -> set:
+        low = _norm(s)
+        fams = set()
+        for fam, syns in ERROR_FAMILY_SYNS.items():
+            if any(k in low for k in syns):
+                fams.add(fam)
+        return fams
+
+    q = _norm(query)
+    q_tokens = [t for t in q.split() if len(t) > 1]
+    q_bi = _ngrams(q_tokens, 2)
+    q_tri = _ngrams(q_tokens, 3)
+    q_fams = _families_for(query)
+
+    lines = _normalize_lines(text)
+    if not lines:
+        return (text or "").strip()
 
-    def _norm(text: str) -> str:
-        t = (text or "").lower()
-        t = re.sub(r"[^\w\s]", " ", t)
-        t = re.sub(r"\s+", " ", t).strip()
-        return t
+    scored: List[Tuple[float, str]] = []
+    for ln in lines:
+        ln_norm = _norm(ln)
+        ln_fams = _families_for(ln)
+
+        fam_overlap = len(q_fams & ln_fams)
+        anchored = 0.0
+        first2 = " ".join(q_tokens[:2]) if len(q_tokens) >= 2 else ""
+        first3 = " ".join(q_tokens[:3]) if len(q_tokens) >= 3 else ""
+        if (first3 and ln_norm.startswith(first3)) or (first2 and ln_norm.startswith(first2)):
+            anchored = 1.0
+
+        bigram_hits = sum(1 for bg in q_bi if bg and bg in ln_norm)
+        trigram_hits = sum(1 for tg in q_tri if tg and tg in ln_norm)
+        token_overlap = sum(1 for t in q_tokens if t and t in ln_norm)
+        exact_phrase = 1.0 if (q and q in ln_norm) else 0.0
+
+        score = (
+            1.70 * fam_overlap +
+            1.00 * anchored +
+            0.80 * trigram_hits +
+            0.55 * bigram_hits +
+            0.40 * exact_phrase +
+            0.30 * token_overlap
+        )
 
-    def _split_sentences(ctx: str) -> List[str]:
-        raw_sents = re.split(r"(?<=[.!?])\s+|\n+|-\s*|\*\s*", ctx or "")
-        return [s.strip() for s in raw_sents if s and len(s.strip()) > 2]
+        if re.match(r"^\s*[-*\u2022]\s*", ln):  # bullet
+            score += 0.10
+        heading = ln_norm.split(":")[0].strip()
+        if heading and (heading in q or (first2 and first2 in heading)):
+            score += 0.15
 
-    ctx = (context or "").strip()
-    if not ctx or not query:
-        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
+        scored.append((score, ln))
 
-    q_norm = _norm(query)
-    q_terms = [t for t in q_norm.split() if len(t) > 2]
-    if not q_terms:
-        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
+    scored.sort(key=lambda x: x[0], reverse=True)
+    top = [ln for s, ln in scored[:max_lines] if s > 0.0]
+    if not top:
+        top = lines[:max_lines]
+    return "\n".join(top).strip()
 
-    sentences = _split_sentences(ctx)
-    matched_exact, matched_any = [], []
-    for s in sentences:
-        s_norm = _norm(s)
-        is_bullet = bool(re.match(r"^[\-\*]\s*", s))
-        overlap = sum(1 for t in q_terms if t in s_norm) + (1 if is_bullet else 0)
-        if overlap >= STRICT_OVERLAP:
-            matched_exact.append(s)
-        elif overlap > 0:
-            matched_any.append(s)
 
-    if matched_exact:
-        kept = matched_exact[:MAX_SENTENCES_STRICT]
-        return "\n".join(kept).strip(), {'mode': 'exact', 'matched_count': len(kept), 'all_sentences': len(sentences)}
+def _friendly_permission_reply(raw: str) -> str:
+    line = (raw or "").strip()
+    line = re.sub(r"^\s*[-*\u2022]\s*", "", line)
+    if not line:
+        return "It looks like you may not have access for this action. Please verify your WMS role/permission with your supervisor or IT."
+    if "verify role access" in line.lower():
+        return "It looks like you may not have access for this action. Please verify your WMS role/permission with your supervisor or IT."
+    if ("permission" in line.lower()) or ("access" in line.lower()) or ("authorization" in line.lower()):
+        return f"It seems to be an access issue: {line}. Please check your role mapping or request access."
+    return line
 
-    if matched_any:
-        kept = matched_any[:MAX_SENTENCES_CONCISE]
-        return "\n".join(kept).strip(), {'mode': 'concise', 'matched_count': len(kept), 'all_sentences': len(sentences)}
-
-    kept = sentences[:MAX_SENTENCES_CONCISE]
-    return "\n".join(kept).strip(), {'mode': 'concise', 'matched_count': 0, 'all_sentences': len(sentences)}
 
-# ------------------------------ Language hint ------------------------------
 def _detect_language_hint(msg: str) -> Optional[str]:
     if re.search(r"[\u0B80-\u0BFF]", msg or ""):  # Tamil
         return "Tamil"
@@ -606,25 +536,26 @@ def _detect_language_hint(msg: str) -> Optional[str]:
         return "Hindi"
     return None
 
-# ------------------------------ Clarifications / tracking ------------------------------
+
 def _build_clarifying_message() -> str:
     return (
         "It seems the issue isn’t resolved yet. Would you like to share a few details so I can check further, "
         "or should I raise a ServiceNow ticket for you?"
     )
 
+
 def _build_tracking_descriptions(issue_text: str, resolved_text: str) -> Tuple[str, str]:
     issue = (issue_text or "").strip()
     resolved = (resolved_text or "").strip()
     short_desc = issue[:100] if issue else (resolved[:100] or "Issue resolved (user confirmation)")
     long_desc = (
-        f'User reported: "{issue}". '
-        f'User confirmation: "{resolved}". '
+        f"User reported: \"{issue}\". "
+        f"User confirmation: \"{resolved}\". "
         f"Tracking record created automatically by NOVA."
     ).strip()
     return short_desc, long_desc
 
-# ------------------------------ Incident helpers ------------------------------
+
 def _is_incident_intent(msg_norm: str) -> bool:
     intent_phrases = [
         "create ticket", "create a ticket", "raise ticket", "raise a ticket", "open ticket", "open a ticket",
@@ -634,6 +565,7 @@ def _is_incident_intent(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in intent_phrases)
 
+
 def _parse_ticket_status_intent(msg_norm: str) -> Dict[str, Optional[str]]:
     status_keywords = ["status", "ticket status", "incident status", "check status", "check ticket status", "check incident status"]
     base_has_status = any(k in msg_norm for k in status_keywords)
@@ -641,7 +573,8 @@ def _parse_ticket_status_intent(msg_norm: str) -> Dict[str, Optional[str]]:
         any(w in msg_norm for w in ("ticket", "incident", "servicenow", "snow")) or
         bool(re.search(r"\binc\d{5,}\b", msg_norm, flags=re.IGNORECASE))
     )
-    if (not base_has_status) or (base_has_status and not has_ticket_marker and any(t in msg_norm for t in DOMAIN_STATUS_TERMS)):
+    # Disambiguation: if it's a domain status query and not clearly ticket/incident, do NOT route to ticket-status.
+    if (not base_has_status) or (base_has_status and not has_ticket_marker and _is_domain_status_context(msg_norm)):
         return {}
     patterns = [
         r"(?:incident\s*id|incidentid|ticket\s*number|number)\s*[:=]?\s*(inc\d+)",
@@ -655,6 +588,7 @@ def _parse_ticket_status_intent(msg_norm: str) -> Dict[str, Optional[str]]:
                 return {"number": val.upper() if val.lower().startswith("inc") else val}
     return {"number": None, "ask_number": True}
 
+
 def _is_resolution_ack_heuristic(msg_norm: str) -> bool:
     phrases = [
         "it is resolved", "resolved", "issue resolved", "problem resolved",
@@ -663,6 +597,7 @@ def _is_resolution_ack_heuristic(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in phrases)
 
+
 def _has_negation_resolved(msg_norm: str) -> bool:
     neg_phrases = [
         "not resolved", "issue not resolved", "still not working", "not working",
@@ -670,11 +605,247 @@ def _has_negation_resolved(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in neg_phrases)
 
+
+def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str, Any]]:
+    STRICT_OVERLAP = 3
+    MAX_SENTENCES_STRICT = 4
+    MAX_SENTENCES_CONCISE = 3
+
+    def _norm(text: str) -> str:
+        t = (text or "").lower()
+        t = re.sub(r"[^\w\s]", " ", t)
+        t = re.sub(r"\s+", " ", t).strip()
+        return t
+
+    def _split_sentences(ctx: str) -> List[str]:
+        raw_sents = re.split(r"(?<=[.!?])\s+|\n+|-\s*|\*\s*", ctx or "")
+        return [s.strip() for s in raw_sents if s and len(s.strip()) > 2]
+
+    ctx = (context or "").strip()
+    if not ctx or not query:
+        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
+    q_norm = _norm(query)
+    q_terms = [t for t in q_norm.split() if len(t) > 2]
+    if not q_terms:
+        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
+    sentences = _split_sentences(ctx)
+    matched_exact, matched_any = [], []
+    for s in sentences:
+        s_norm = _norm(s)
+        is_bullet = bool(re.match(r"^[\-\*]\s*", s))
+        overlap = sum(1 for t in q_terms if t in s_norm) + (1 if is_bullet else 0)
+        if overlap >= STRICT_OVERLAP:
+            matched_exact.append(s)
+        elif overlap > 0:
+            matched_any.append(s)
+    if matched_exact:
+        kept = matched_exact[:MAX_SENTENCES_STRICT]
+        return "\n".join(kept).strip(), {'mode': 'exact', 'matched_count': len(kept), 'all_sentences': len(sentences)}
+    if matched_any:
+        kept = matched_any[:MAX_SENTENCES_CONCISE]
+        return "\n".join(kept).strip(), {'mode': 'concise', 'matched_count': len(kept), 'all_sentences': len(sentences)}
+    kept = sentences[:MAX_SENTENCES_CONCISE]
+    return "\n".join(kept).strip(), {'mode': 'concise', 'matched_count': 0, 'all_sentences': len(sentences)}
+
+
+def _extract_errors_only(text: str, max_lines: int = 12) -> str:
+    """
+    Collect error bullets/heading-style lines from the SOP errors section.
+    Generic: keeps bullet points (•, -, *), and lines that look like "Heading: details".
+    This ensures items like 'ASN not found', 'Trailer status locked', etc., are preserved.
+    """
+    kept: List[str] = []
+    for ln in _normalize_lines(text):
+        if re.match(r"^\s*[-*\u2022]\s*", ln) or (":" in ln):
+            kept.append(ln)
+            if len(kept) >= max_lines:
+                break
+    return "\n".join(kept).strip() if kept else (text or "").strip()
+
+
+def _filter_permission_lines(text: str, max_lines: int = 6) -> str:
+    PERM_SYNONYMS = (
+        "permission", "permissions", "access", "authorization", "authorisation",
+        "role", "role mapping", "security profile", "not allowed", "not authorized", "denied", "insufficient"
+    )
+    kept: List[str] = []
+    for ln in _normalize_lines(text):
+        low = ln.lower()
+        if any(k in low for k in PERM_SYNONYMS):
+            kept.append(ln)
+        if len(kept) >= max_lines:
+            break
+    return "\n".join(kept).strip() if kept else (text or "").strip()
+
+
+def _extract_escalation_line(text: str) -> Optional[str]:
+    if not text:
+        return None
+    lines = _normalize_lines(text)
+    if not lines:
+        return None
+    start_idx = None
+    for i, ln in enumerate(lines):
+        low = ln.lower()
+        if "escalation" in low or "escalation path" in low or "escalate" in low:
+            start_idx = i
+            break
+    block = []
+    if start_idx is not None:
+        for j in range(start_idx, min(len(lines), start_idx + 6)):
+            if not lines[j].strip():
+                break
+            block.append(lines[j].strip())
+    else:
+        block = [ln.strip() for ln in lines if ("->" in ln or "→" in ln)]
+    if not block:
+        return None
+    text_block = " ".join(block)
+    m = re.search(r"escalation[^:]*:\s*(.+)", text_block, flags=re.IGNORECASE)
+    path = m.group(1).strip() if m else None
+    if not path:
+        arrow_lines = [ln for ln in block if ("->" in ln or "→" in ln)]
+        if arrow_lines:
+            path = arrow_lines[0]
+    if not path:
+        m2 = re.search(r"(operator.*?administrator|operator.*)", text_block, flags=re.IGNORECASE)
+        path = m2.group(1).strip() if m2 else None
+    if not path:
+        return None
+    path = path.replace("->", "→").strip()
+    path = re.sub(r"^(?i:escalation\s*path)\s*:\s*", "", path).strip()
+    return f"If you want to escalate the issue, follow: {path}"
+
+
+def _classify_resolution_llm(user_message: str) -> bool:
+    if not GEMINI_API_KEY:
+        return False
+    prompt = f"""Classify if the following user message indicates that the issue is resolved or working now.
+Return only 'true' or 'false'.
+Message: {user_message}"""
+    headers = {"Content-Type": "application/json"}
+    payload = {"contents": [{"parts": [{"text": prompt}]}]}
+    try:
+        resp = requests.post(GEMINI_URL, headers=headers, json=payload, timeout=12, verify=GEMINI_SSL_VERIFY)
+        data = resp.json()
+        text = (
+            data.get("candidates", [{}])[0]
+              .get("content", {})
+              .get("parts", [{}])[0]
+              .get("text", "")
+        )
+        return "true" in (text or "").strip().lower()
+    except Exception:
+        return False
+
+
+def _set_incident_resolved(sys_id: str) -> bool:
+    try:
+        token = get_valid_token()
+        instance_url = os.getenv("SERVICENOW_INSTANCE_URL")
+        if not instance_url:
+            print("[SN PATCH resolve] missing SERVICENOW_INSTANCE_URL")
+            return False
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+        url = f"{instance_url}/api/now/table/incident/{sys_id}"
+        close_code_val = os.getenv("SERVICENOW_CLOSE_CODE", "Solution provided")
+        close_notes_val = os.getenv("SERVICENOW_RESOLUTION_NOTES", "Issue resolved, user confirmed")
+        caller_sysid = os.getenv("SERVICENOW_CALLER_SYSID")
+        resolved_by_sysid = os.getenv("SERVICENOW_RESOLVED_BY_SYSID")
+        assign_group = os.getenv("SERVICENOW_ASSIGNMENT_GROUP_SYSID")
+        require_progress = os.getenv("SERVICENOW_REQUIRE_IN_PROGRESS_FIRST", "false").lower() in ("1", "true", "yes")
+        if require_progress:
+            try:
+                resp1 = requests.patch(url, headers=headers, json={"state": "2"}, verify=VERIFY_SSL, timeout=25)
+                print(f"[SN PATCH progress] status={resp1.status_code} body={resp1.text[:500]}")
+            except Exception as e:
+                print(f"[SN PATCH progress] exception={safe_str(e)}")
+
+        def clean(d: dict) -> dict:
+            return {k: v for k, v in d.items() if v is not None}
+
+        payload_A = clean({
+            "state": "6",
+            "close_code": close_code_val,
+            "close_notes": close_notes_val,
+            "caller_id": caller_sysid,
+            "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
+            "work_notes": "Auto-resolve set by NOVA.",
+            "resolved_by": resolved_by_sysid,
+            "assignment_group": assign_group,
+        })
+        respA = requests.patch(url, headers=headers, json=payload_A, verify=VERIFY_SSL, timeout=25)
+        if respA.status_code in (200, 204):
+            return True
+        print(f"[SN PATCH resolve A] status={respA.status_code} body={respA.text[:500]}")
+
+        payload_B = clean({
+            "state": "Resolved",
+            "close_code": close_code_val,
+            "close_notes": close_notes_val,
+            "caller_id": caller_sysid,
+            "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
+            "work_notes": "Auto-resolve set by NOVA.",
+            "resolved_by": resolved_by_sysid,
+            "assignment_group": assign_group,
+        })
+        respB = requests.patch(url, headers=headers, json=payload_B, verify=VERIFY_SSL, timeout=25)
+        if respB.status_code in (200, 204):
+            return True
+        print(f"[SN PATCH resolve B] status={respB.status_code} body={respB.text[:500]}")
+
+        code_field = os.getenv("SERVICENOW_RESOLUTION_CODE_FIELD", "close_code")
+        notes_field = os.getenv("SERVICENOW_RESOLUTION_NOTES_FIELD", "close_notes")
+        payload_C = clean({
+            "state": "6",
+            code_field: close_notes_val,  # (if you have custom fields, adjust here)
+            notes_field: close_notes_val,
+            "caller_id": caller_sysid,
+            "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
+            "work_notes": "Auto-resolve set by NOVA.",
+            "resolved_by": resolved_by_sysid,
+            "assignment_group": assign_group,
+        })
+        respC = requests.patch(url, headers=headers, json=payload_C, verify=VERIFY_SSL, timeout=25)
+        if respC.status_code in (200, 204):
+            return True
+        print(f"[SN PATCH resolve C] status={respC.status_code} body={respC.text[:500]}")
+        return False
+    except Exception as e:
+        print(f"[SN PATCH resolve] exception={safe_str(e)}")
+        return False
+
+
+# ------------------------------ Prereq helper ------------------------------
+def _find_prereq_section_text(best_doc: str) -> str:
+    """
+    Return the prerequisites section text, trying common heading variants.
+    Generic for future SOPs—no document-specific keywords.
+    """
+    variants = [
+        "Pre-Requisites",
+        "Prerequisites",
+        "Pre Requisites",
+        "Pre-Requirements",
+        "Requirements",
+    ]
+    for title in variants:
+        txt = get_section_text(best_doc, title)
+        if txt and txt.strip():
+            return txt.strip()
+    return ""
+
+
 # ------------------------------ Health ------------------------------
 @app.get("/")
 async def health_check():
     return {"status": "ok"}
 
+
 # ------------------------------ Chat ------------------------------
 @app.post("/chat")
 async def chat_with_ai(input_data: ChatInput):
@@ -702,31 +873,10 @@ async def chat_with_ai(input_data: ChatInput):
             }
 
         # Resolution ack (auto incident + mark Resolved)
-        is_llm_resolved = False
-        if GEMINI_API_KEY:
-            try:
-                prompt = f"Classify if resolved: return 'true' or 'false'.\nMessage: {input_data.user_message}"
-                headers = {"Content-Type": "application/json"}
-                payload = {"contents": [{"parts": [{"text": prompt}]}]}
-                resp = requests.post(GEMINI_URL, headers=headers, json=payload, timeout=12, verify=GEMINI_SSL_VERIFY)
-                data = resp.json()
-                text = (
-                    data.get("candidates", [{}])[0]
-                    .get("content", {})
-                    .get("parts", [{}])[0]
-                    .get("text", "")
-                )
-                is_llm_resolved = "true" in (text or "").strip().lower()
-            except Exception:
-                is_llm_resolved = False
-
+        is_llm_resolved = _classify_resolution_llm(input_data.user_message)
         if _has_negation_resolved(msg_norm):
             is_llm_resolved = False
-        if (not _has_negation_resolved(msg_norm)) and (any(p in msg_norm for p in [
-            "it is resolved", "resolved", "issue resolved", "problem resolved",
-            "it's working", "working now", "works now", "fixed", "sorted",
-            "ok now", "fine now", "all good", "all set", "thanks works", "thank you it works", "back to normal",
-        ]) or is_llm_resolved):
+        if (not _has_negation_resolved(msg_norm)) and (_is_resolution_ack_heuristic(msg_norm) or is_llm_resolved):
             try:
                 short_desc, long_desc = _build_tracking_descriptions(input_data.last_issue, input_data.user_message)
                 result = create_incident(short_desc, long_desc)
@@ -735,28 +885,8 @@ async def chat_with_ai(input_data: ChatInput):
                     sys_id = result.get("sys_id")
                     resolved_note = ""
                     if sys_id:
-                        # attempt to mark resolved
-                        try:
-                            token = get_valid_token()
-                            instance_url = os.getenv("SERVICENOW_INSTANCE_URL")
-                            if instance_url and token:
-                                headers = {
-                                    "Authorization": f"Bearer {token}",
-                                    "Accept": "application/json",
-                                    "Content-Type": "application/json",
-                                }
-                                url = f"{instance_url}/api/now/table/incident/{sys_id}"
-                                payload_A = {
-                                    "state": "6",
-                                    "close_code": os.getenv("SERVICENOW_CLOSE_CODE", "Solution provided"),
-                                    "close_notes": os.getenv("SERVICENOW_RESOLUTION_NOTES", "Issue resolved, user confirmed"),
-                                }
-                                respA = requests.patch(url, headers=headers, json=payload_A, verify=VERIFY_SSL, timeout=25)
-                                resolved_note = " (marked Resolved)" if respA.status_code in (200, 204) else " (could not mark Resolved; please update manually)"
-                            else:
-                                resolved_note = ""
-                        except Exception:
-                            resolved_note = ""
+                        ok = _set_incident_resolved(sys_id)
+                        resolved_note = " (marked Resolved)" if ok else " (could not mark Resolved; please update manually)"
                     return {
                         "bot_response": f"✅ Incident created: {inc_number}{resolved_note}",
                         "status": "OK",
@@ -816,7 +946,7 @@ async def chat_with_ai(input_data: ChatInput):
                 "debug": {"intent": "create_ticket"},
             }
 
-        # Status intent (ticket/incident)
+        # Status intent (ticket/incident) — disambiguated
         status_intent = _parse_ticket_status_intent(msg_norm)
         if status_intent:
             if status_intent.get("ask_number"):
@@ -876,7 +1006,6 @@ async def chat_with_ai(input_data: ChatInput):
         metadatas = kb_results.get("metadatas", [])
         distances = kb_results.get("distances", [])
         combined = kb_results.get("combined_scores", [])
-
         items: List[Dict[str, Any]] = []
         for i, doc in enumerate(documents):
             text = doc.strip() if isinstance(doc, str) else ""
@@ -895,10 +1024,11 @@ async def chat_with_ai(input_data: ChatInput):
         selected = items[:max(1, 2)]
         context_raw = "\n\n---\n\n".join([s["text"] for s in selected]) if selected else ""
 
+        # Compute filter info for gating only; do NOT use the filtered text for steps
         filtered_text, filt_info = _filter_context_for_query(context_raw, input_data.user_message)
         filtered_context = filtered_text
 
-        context = context_raw
+        context = context_raw  # keep raw; we'll decide below
         context_found = bool(context.strip())
         best_distance = min([d for d in distances if d is not None], default=None) if distances else None
         best_combined = max([c for c in combined if c is not None], default=None) if combined else None
@@ -907,18 +1037,18 @@ async def chat_with_ai(input_data: ChatInput):
         top_meta = (metadatas or [{}])[0] if metadatas else {}
         msg_low = (input_data.user_message or "").lower()
 
-        # Force "steps" intent when user asks for next step
-        if _detect_next_intent(input_data.user_message):
-            detected_intent = "steps"
-
         GENERIC_ERROR_TERMS = ("error", "issue", "problem", "not working", "failed", "failure")
         generic_error_signal = any(t in msg_low for t in GENERIC_ERROR_TERMS)
 
-        PREREQ_TERMS = ("pre req", "pre-requisite", "pre-requisites", "prerequisite",
-                        "prerequisites", "pre requirement", "pre-requirements", "requirements")
+        # Query-based prereq nudge
+        PREREQ_TERMS = (
+            "pre req", "pre-requisite", "pre-requisites", "prerequisite",
+            "prerequisites", "pre requirement", "pre-requirements", "requirements"
+        )
         if detected_intent == "neutral" and any(t in msg_low for t in PREREQ_TERMS):
             detected_intent = "prereqs"
 
+        # Force errors intent for permissions
         PERM_QUERY_TERMS = [
             "permission", "permissions", "access", "access right", "authorization", "authorisation",
             "role", "role access", "security", "security profile", "privilege", "not allowed", "not authorized", "denied",
@@ -927,11 +1057,16 @@ async def chat_with_ai(input_data: ChatInput):
         if is_perm_query:
             detected_intent = "errors"
 
+        # Heading-aware prereq nudge
         sec_title = ((top_meta or {}).get("section") or "").strip().lower()
-        PREREQ_HEADINGS = ("pre-requisites", "prerequisites", "pre requisites", "pre-requirements", "requirements")
+        PREREQ_HEADINGS = (
+            "pre-requisites", "prerequisites", "pre requisites",
+            "pre-requirements", "requirements"
+        )
         if detected_intent == "neutral" and any(h in sec_title for h in PREREQ_HEADINGS):
             detected_intent = "prereqs"
 
+        # --- Module-aware steps nudge (appointments, picking, shipping, etc.) ---
         STEPS_TERMS = ("how to", "procedure", "perform", "steps", "do", "navigate")
         ACTIONS_PRESENT = any(s in msg_low for syns in ACTION_SYNONYMS_EXT.values() for s in syns)
         mod_tags = ((top_meta or {}).get("module_tags") or "").lower()
@@ -947,6 +1082,7 @@ async def chat_with_ai(input_data: ChatInput):
         if detected_intent in ("neutral", "prereqs") and looks_like_steps_query and looks_like_module:
             detected_intent = "steps"
 
+        # --- Meaning-aware SOP gating (uses filter info) ---
         def _contains_any(s: str, keywords: tuple) -> bool:
             low = (s or "").lower()
             return any(k in low for k in keywords)
@@ -1014,6 +1150,97 @@ async def chat_with_ai(input_data: ChatInput):
                     return act
             return None
 
+        def _strip_boilerplate(raw_context: str) -> str:
+            """Remove document title/date/author/change-history noise from steps."""
+            MONTH_TERMS = ("january", "february", "march", "april", "may", "june",
+                           "july", "august", "september", "october", "november", "december")
+            lines = _normalize_lines(raw_context)
+            cleaned: List[str] = []
+            for ln in lines:
+                low = ln.lower()
+                is_change_hist = ("change history" in low) or ("initial draft" in low) or ("review" in low) or ("version" in low)
+                has_month_year = any(m in low for m in MONTH_TERMS) and bool(re.search(r"\b20\d{2}\b", low))
+                is_title_line = ("sop document" in low) or ("contents" in low)
+                if is_change_hist or has_month_year or is_title_line:
+                    continue
+                cleaned.append(ln)
+            return "\n".join(cleaned).strip()
+
+        def _extract_action_block(raw_context: str, target_act: Optional[str]) -> str:
+            """
+            Extract the contiguous block of lines for the target action (create/update/delete).
+            Start when a line mentions the target action OR looks procedural for 'create',
+            and stop ONLY when a line is a clear boundary:
+              - inline heading for another topic (e.g., 'Appointment schedule updation', 'Appointment deletion'), OR
+              - a line that strongly signals a different action (update/delete) via extended synonyms.
+            """
+            if not raw_context.strip():
+                return raw_context
+
+            lines = _normalize_lines(raw_context)
+            if not lines or not target_act:
+                return raw_context
+
+            INLINE_BOUNDARIES = (
+                "appointment schedule updation",
+                "schedule updation",
+                "appointment deletion",
+                "deletion",
+            )
+
+            other_terms: List[str] = []
+            for act, syns in ACTION_SYNONYMS_EXT.items():
+                if act != target_act:
+                    other_terms.extend(syns)
+            other_terms_low = set(t.lower() for t in other_terms)
+
+            def is_boundary_line(low: str) -> bool:
+                if any(h in low for h in INLINE_BOUNDARIES):
+                    return True
+                if any(t in low for t in other_terms_low):
+                    return True
+                return False
+
+            PROCEDURAL_VERBS = ("select", "choose", "click", "open", "add", "assign", "save",
+                                "navigate", "tag", "displayed", "triggered")
+            def is_procedural(low: str) -> bool:
+                return any(v in low for v in PROCEDURAL_VERBS)
+
+            target_terms_low = set(t.lower() for t in ACTION_SYNONYMS_EXT.get(target_act, []))
+
+            started = False
+            block: List[str] = []
+
+            for ln in lines:
+                low = ln.lower()
+
+                contains_target = any(t in low for t in target_terms_low)
+                if not started:
+                    if contains_target or (target_act == "create" and is_procedural(low)):
+                        started = True
+                        block.append(ln)
+                    continue
+
+                if is_boundary_line(low):
+                    break
+
+                block.append(ln)
+
+            return "\n".join(block).strip() if block else raw_context
+
+        def _filter_steps_by_action(raw_context: str, asked_act: Optional[str]) -> str:
+            cleaned = _strip_boilerplate(raw_context)
+            block = _extract_action_block(cleaned, asked_act)
+            if asked_act:
+                other_terms: List[str] = []
+                for act, syns in ACTION_SYNONYMS_EXT.items():
+                    if act != asked_act:
+                        other_terms.extend(syns)
+                lines = _normalize_lines(block)
+                lines = [ln for ln in lines if not any(t in ln.lower() for t in other_terms)]
+                block = "\n".join(lines).strip() if lines else block
+            return block
+
         escalation_line = None
         full_errors = None
         next_step_applied = False
@@ -1023,55 +1250,49 @@ async def chat_with_ai(input_data: ChatInput):
             context_preformatted = False
             full_steps = None
 
-            # Detect asked action generically using ACTION_SYNONYMS_EXT (already defined)
-            asked_action = _detect_action_from_query(input_data.user_message)  # 'create'/'update'/'delete'/None
-
-            # 1) Try KB-tagged action section (existing)
+            # 1) Try by KB action tags
             action_steps = _get_steps_for_action(best_doc, kb_results.get("actions", []))
             if action_steps:
                 full_steps = action_steps
             else:
-                # 2) Global action lookup (existing)
                 if kb_results.get("actions"):
-                    alt_doc, alt_steps = _get_steps_for_action_global(kb_results["actions"], prefer_doc=best_doc)
-                    if alt_steps:
-                        best_doc = alt_doc
-                        full_steps = alt_steps
-
-                # 3) Generic: exact section by meta.action_tag in SAME best_doc
-                if not full_steps and asked_action:
-                    via_tag_steps = _get_steps_by_action_tag(best_doc, asked_action)
-                    if via_tag_steps:
-                        full_steps = via_tag_steps
-
-                # 4) Prefer requested action when defaulting
-                if not full_steps:
-                    default_sec = _pick_default_action_section_with_preference(best_doc, asked_action)
-                    if default_sec:
-                        full_steps = get_section_text(best_doc, default_sec)
+                   alt_doc, alt_steps = _get_steps_for_action_global(kb_results["actions"], prefer_doc=best_doc)
+                   if alt_steps:
+                      best_doc = alt_doc   # switch to the doc that actually has the section
+                      full_steps = alt_steps
+                       
+                asked_action = _detect_action_from_query(input_data.user_message)
+                if not kb_results.get("actions") and asked_action:
+                   alt_doc, alt_steps = _get_steps_for_action_global([asked_action], prefer_doc=best_doc)
+                   if alt_steps:
+                      best_doc = alt_doc   # switch to the doc that actually has the section
+                      full_steps = alt_steps
 
-                # 5) Existing fallbacks
                 if not full_steps:
-                    full_steps = get_best_steps_section_text(best_doc)
+                   default_sec = _pick_default_action_section(best_doc)
+                   if default_sec:
+                      full_steps = get_section_text(best_doc, default_sec)
                 if not full_steps:
-                    sec = (top_meta or {}).get("section")
-                    if sec:
-                        full_steps = get_section_text(best_doc, sec)
-
+                   full_steps = get_best_steps_section_text(best_doc)
+                if not full_steps:
+                   sec = (top_meta or {}).get("section")
+                   if sec:
+                      full_steps = get_section_text(best_doc, sec)
+    
             if full_steps:
-                # Generic boundary cutter (prevents create/update/delete bleed)
-                full_steps = _cut_at_next_boundary_generic(full_steps, best_doc, asked_action)
-
-                # Scope 'Save' lines to current section
-                save_local = _find_save_lines_in_section(full_steps, max_lines=2)
-                if save_local:
+                # Always add Save lines if present anywhere in the doc (independent of query wording)
+                save_lines = _find_save_lines_in_doc(best_doc, max_lines=2)
+                if save_lines:
                     low_steps = (full_steps or "").lower()
                     if not any(s in low_steps for s in SAVE_SYNS):
-                        full_steps = (full_steps or "").rstrip() + "\n" + save_local
+                        full_steps = (full_steps or "").rstrip() + "\n" + save_lines
+
+                asked_action = _detect_action_from_query(input_data.user_message)
+                full_steps = _filter_steps_by_action(full_steps, asked_action)
 
-                # Number + Next steps
                 numbered_full = _ensure_numbering(full_steps)
-                next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.30)
+                next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
+
                 if next_only is not None:
                     if len(next_only) == 0:
                         context = "You are at the final step of this SOP. No further steps."
@@ -1087,8 +1308,9 @@ async def chat_with_ai(input_data: ChatInput):
                     context = full_steps
                     context_preformatted = False
 
-                context_found = True
+                # Clear filter info for debug clarity
                 filt_info = {'mode': None, 'matched_count': None, 'all_sentences': None}
+                context_found = True
 
         elif best_doc and detected_intent == "errors":
             full_errors = get_best_errors_section_text(best_doc)
@@ -1112,21 +1334,27 @@ async def chat_with_ai(input_data: ChatInput):
             else:
                 full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
                 if full_steps:
+                    asked_action = _detect_action_from_query(input_data.user_message)
+                    full_steps = _filter_steps_by_action(full_steps, asked_action)
                     context = full_steps
                     detected_intent = "steps"
                     context_preformatted = False
 
         elif best_doc and detected_intent == "prereqs":
-            full_prereqs = get_section_text(best_doc, "Pre-Requisites") or get_section_text(best_doc, "Prerequisites")
+            full_prereqs = _find_prereq_section_text(best_doc)
             if full_prereqs:
                 context = full_prereqs.strip()
             else:
                 full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
                 if full_steps:
+                    asked_action = _detect_action_from_query(input_data.user_message)
+                    full_steps = _filter_steps_by_action(full_steps, asked_action)
                     context = full_steps
                     detected_intent = "steps"
                     context_preformatted = False
+
         else:
+            # Neutral or other intents: use filtered context
             context = filtered_context
 
         language_hint = _detect_language_hint(input_data.user_message)
@@ -1161,19 +1389,23 @@ Return ONLY the rewritten guidance."""
             except Exception:
                 bot_text, http_code = "", 0
 
+        # Deterministic local formatting
         if detected_intent == "steps":
             if ('context_preformatted' in locals()) and context_preformatted:
                 bot_text = context
             else:
                 bot_text = _ensure_numbering(context)
+
         elif detected_intent == "errors":
             if not bot_text.strip() or http_code == 429:
                 bot_text = context.strip()
             if escalation_line:
                 bot_text = (bot_text or "").rstrip() + "\n\n" + escalation_line
+
         else:
             bot_text = context
 
+        # If the user explicitly asks to escalate, append escalation even in 'steps' intent
         needs_escalation = (" escalate" in msg_norm) or ("escalation" in msg_norm)
         if needs_escalation and best_doc:
             esc_text = get_escalation_text(best_doc)
@@ -1183,6 +1415,7 @@ Return ONLY the rewritten guidance."""
             if line:
                 bot_text = (bot_text or "").rstrip() + "\n\n" + line
 
+        # Guarantee non-empty bot response
         if not (bot_text or "").strip():
             if context.strip():
                 bot_text = context.strip()
@@ -1192,6 +1425,7 @@ Return ONLY the rewritten guidance."""
                     "Share a bit more detail (module/screen/error), or say ‘create ticket’."
                 )
 
+        # Status: mark OK when we served steps successfully
         if detected_intent == "steps" and bot_text.strip():
             status = "OK"
         else:
@@ -1234,6 +1468,7 @@ Return ONLY the rewritten guidance."""
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
+
 # ------------------------------ Ticket description generation ------------------------------
 @app.post("/generate_ticket_desc")
 async def generate_ticket_desc_ep(input_data: TicketDescInput):
@@ -1272,6 +1507,7 @@ async def generate_ticket_desc_ep(input_data: TicketDescInput):
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
+
 # ------------------------------ Incident status ------------------------------
 @app.post("/incident_status")
 async def incident_status(input_data: TicketStatusInput):
@@ -1312,6 +1548,7 @@ async def incident_status(input_data: TicketStatusInput):
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
+
 # ------------------------------ Incident ------------------------------
 @app.post("/incident")
 async def raise_incident(input_data: IncidentInput):
@@ -1322,25 +1559,8 @@ async def raise_incident(input_data: IncidentInput):
             sys_id = result.get("sys_id")
             resolved_note = ""
             if bool(input_data.mark_resolved) and sys_id not in ("<unknown>", None):
-                try:
-                    token = get_valid_token()
-                    instance_url = os.getenv("SERVICENOW_INSTANCE_URL")
-                    if instance_url and token:
-                        headers = {
-                            "Authorization": f"Bearer {token}",
-                            "Accept": "application/json",
-                            "Content-Type": "application/json",
-                        }
-                        url = f"{instance_url}/api/now/table/incident/{sys_id}"
-                        payload_A = {
-                            "state": "6",
-                            "close_code": os.getenv("SERVICENOW_CLOSE_CODE", "Solution provided"),
-                            "close_notes": os.getenv("SERVICENOW_RESOLUTION_NOTES", "Issue resolved, user confirmed"),
-                        }
-                        respA = requests.patch(url, headers=headers, json=payload_A, verify=VERIFY_SSL, timeout=25)
-                        resolved_note = " (marked Resolved)" if respA.status_code in (200, 204) else " (could not mark Resolved; please update manually)"
-                except Exception:
-                    resolved_note = ""
+                ok = _set_incident_resolved(sys_id)
+                resolved_note = " (marked Resolved)" if ok else " (could not mark Resolved; please update manually)"
             ticket_text = f"Incident created: {inc_number}{resolved_note}" if inc_number else "Incident created."
             return {
                 "bot_response": f"✅ {ticket_text}",