Update services/kb_creation.py

services/kb_creation.py

@@ -1,33 +1,22 @@
 
-# services/kb_creation.py (DROP-IN REPLACEMENT)
-# Generic (heading-agnostic) sectionizing + robust intent/action tagging,
-# with hybrid retrieval tuned for SOPs that mix headings or omit them.
-
+# services/kb_creation.py
 import os
 import re
 import pickle
-from typing import TYPE_CHECKING, Any, List, Dict, Tuple, Optional
-
-# Word parsing
-if TYPE_CHECKING:
-    from docx import Document as DocxDocument
-else:
-    DocxDocument = Any
-try:
-    from docx import Document  # runtime import
-except Exception:
-    Document = None  # type: ignore
-
-# Vector DB / embeddings
-import chromadb  # type: ignore
-from sentence_transformers import SentenceTransformer  # type: ignore
+from typing import List, Dict, Any, Tuple, Optional
+from docx import Document
+from sentence_transformers import SentenceTransformer
+import chromadb
 
+# ------------------------------ ChromaDB setup ------------------------------
 CHROMA_PATH = os.path.join(os.getcwd(), "chroma_db")
 client = chromadb.PersistentClient(path=CHROMA_PATH)
 collection = client.get_or_create_collection(name="knowledge_base")
+
+# ------------------------------ Embedding model ------------------------------
 model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 
-# ----------------------------- BM25 structures -----------------------------
+# ------------------------------ BM25 (lightweight) ------------------------------
 BM25_INDEX_FILE = os.path.join(CHROMA_PATH, "bm25_index.pkl")
 bm25_docs: List[Dict[str, Any]] = []
 bm25_inverted: Dict[str, List[int]] = {}
@@ -37,7 +26,7 @@ bm25_ready: bool = False
 BM25_K1 = 1.5
 BM25_B = 0.75
 
-# ----------------------------- Token utilities -----------------------------
+# ------------------------------ Utilities ------------------------------
 def _tokenize(text: str) -> List[str]:
     if not text:
         return []
@@ -53,10 +42,34 @@ def _normalize_query(q: str) -> str:
 def _tokenize_meta_value(val: Optional[str]) -> List[str]:
     return _tokenize(val or "")
 
-# ----------------------------- Line splitting / chunking -----------------------------
-BULLET_RE = re.compile(r"^\s*(?:[-*\u2022]|\d+[.)])\s+", re.IGNORECASE)
+# ------------------------------ DOCX parsing & chunking ------------------------------
+BULLET_RE = re.compile(r"^\s*(?:[\-\*\u2022]|\d+[.)])\s+", re.IGNORECASE)
+
+def _split_by_sections(doc: Document) -> List[Tuple[str, List[str]]]:
+    sections: List[Tuple[str, List[str]]] = []
+    current_title = None
+    current_paras: List[str] = []
+    for para in doc.paragraphs:
+        text = (para.text or "").strip()
+        style_name = (para.style.name if para.style else "") or ""
+        is_heading = bool(re.match(r"Heading\s*\d+", style_name, flags=re.IGNORECASE))
+        if is_heading and text:
+            if current_title or current_paras:
+                sections.append((current_title or "Untitled Section", current_paras))
+            current_title = text
+            current_paras = []
+        else:
+            if text:
+                current_paras.append(text)
+    if current_title or current_paras:
+        sections.append((current_title or "Untitled Section", current_paras))
+    if not sections:
+        all_text = [p.text.strip() for p in doc.paragraphs if p.text and p.text.strip()]
+        sections = [("Document", all_text)]
+    return sections
 
 def _paragraphs_to_lines(paragraphs: List[str]) -> List[str]:
+    """Preserve bullets/numbered list lines; split long paragraphs by sentence boundaries."""
     lines: List[str] = []
     for p in (paragraphs or []):
         p = (p or "").strip()
@@ -65,21 +78,19 @@ def _paragraphs_to_lines(paragraphs: List[str]) -> List[str]:
         if BULLET_RE.match(p):
             lines.append(p)
             continue
-        # split on sentence ends
         parts = [s.strip() for s in re.split(r"(?<=[.!?])\s+", p) if s.strip()]
         lines.extend(parts)
     return lines
 
-def _chunk_text_with_context(doc_title: str, section_title: str, lines_or_paras: List[str], max_words: int = 160) -> List[str]:
-    """Chunk a list of lines or paragraphs into ~max_words, keeping bullets and step starts intact."""
-    lines = _paragraphs_to_lines(lines_or_paras)
+def _chunk_text_with_context(doc_title: str, section_title: str, paragraphs: List[str], max_words: int = 160) -> List[str]:
+    """Smaller chunks (~160 words), bullet-aware."""
+    lines = _paragraphs_to_lines(paragraphs)
     chunks: List[str] = []
     current: List[str] = []
     current_len = 0
     for ln in lines:
         w = ln.split()
-        # boundary when adding would exceed or bullet encounters and we already have something
-        if current and (current_len + len(w) > max_words or BULLET_RE.match(ln)):
+        if current_len + len(w) > max_words or (BULLET_RE.match(ln) and current):
             chunk = " ".join(current).strip()
             if chunk:
                 chunks.append(chunk)
@@ -98,160 +109,16 @@ def _chunk_text_with_context(doc_title: str, section_title: str, lines_or_paras:
             chunks = [body]
     return chunks
 
-# ----------------------------- Generic intent & action inference -----------------------------
-PROCEDURE_VERBS = {
-    "navigate", "select", "scan", "verify", "confirm", "print", "move", "complete",
-    "click", "open", "choose", "enter", "update", "save", "delete", "create",
-    "attach", "assign", "reschedule", "edit", "change", "cancel", "remove", "tag"
-}
-ERROR_TERMS = {
-    "error", "fail", "failure", "not working", "cannot", "can't", "mismatch",
-    "locked", "permission", "permissions", "access denied", "not authorized",
-    "authorised", "insufficient", "no access", "timeout", "short pick",
-    "not found", "missing", "unavailable"
-}
-PREREQ_TERMS = {"pre-requisite", "prerequisites", "requirements", "pre requirements", "pre requisites"}
-PURPOSE_TERMS = {"purpose", "overview", "introduction"}
-ESCALATION_MARKERS = {"escalation", "escalation path", "→", "->"}
-
-ACTION_SYNS = {
-    "create": {"create", "creation", "add", "new", "generate", "tag"},
-    "update": {"update", "modify", "change", "edit", "reschedule", "re-schedule", "updation"},
-    "delete": {"delete", "remove", "cancel", "deletion", "unassign"}
-}
-
-STEP_NUM_PATTERNS = [
-    re.compile(r"^\s*\d+\s*[.)]\s+"),     # 1. / 1)
-    re.compile(r"^\s*[\u2460-\u2473]\s+"),# ① ② ...
-    re.compile(r"^\s*[-*\u2022]\s+"),     # - * •
-    re.compile(r"(?i)\bstep\s*\d+\s*:")   # Step 1:
-]
-
-def _norm(s: str) -> str:
-    s = (s or "").lower()
-    s = re.sub(r"[^\w\s→\-]", " ", s)  # keep arrows
-    s = re.sub(r"\s+", " ", s).strip()
-    return s
-
-def line_signals(line: str) -> Dict[str, float]:
-    """Return a score dictionary for intent signals on a single line."""
-    low = _norm(line)
-    sig = {"steps": 0.0, "errors": 0.0, "prereqs": 0.0, "purpose": 0.0, "escalation": 0.0}
-    # Steps signal
-    if any(p.search(line) for p in STEP_NUM_PATTERNS): sig["steps"] += 1.2
-    if any(v in low for v in PROCEDURE_VERBS): sig["steps"] += 0.8
-    # Errors signal
-    if any(t in low for t in ERROR_TERMS): sig["errors"] += 1.1
-    if ":" in line and len(line.split(":")[0]) <= 80: sig["errors"] += 0.2  # 'Heading: details'
-    # Prereqs
-    if any(t in low for t in PREREQ_TERMS): sig["prereqs"] += 1.1
-    # Purpose
-    if any(t in low for t in PURPOSE_TERMS): sig["purpose"] += 0.9
-    # Escalation
-    if any(m in low for m in ESCALATION_MARKERS): sig["escalation"] += 1.0
-    return sig
-
-def dominant_intent(block: List[str]) -> str:
-    """Return the dominant intent tag for a block of lines."""
-    agg = {"steps": 0.0, "errors": 0.0, "prereqs": 0.0, "purpose": 0.0, "escalation": 0.0}
-    for ln in block:
-        s = line_signals(ln)
-        for k, v in s.items():
-            agg[k] += v
-    order = ["steps", "errors", "prereqs", "escalation", "purpose"]
-    best = max(order, key=lambda k: agg[k])
-    if max(agg.values()) < 0.25:
-        return "neutral"
-    return best
-
-def infer_action(block: List[str]) -> str:
-    """Infer action_tag (create/update/delete) from content of the block."""
-    counts = {"create": 0, "update": 0, "delete": 0}
-    for ln in block:
-        low = _norm(ln)
-        for act, syns in ACTION_SYNS.items():
-            if any(s in low for s in syns):
-                counts[act] += 1
-    act = max(counts, key=lambda k: counts[k])
-    return act if counts[act] > 0 else ""
-
-def is_boundary_to_new_section(prev_intent: str, curr_intent: str, ln: str) -> bool:
-    """Signal to start a new synthetic section."""
-    low = _norm(ln)
-    # Dominant intent flips (e.g., steps → errors)
-    if prev_intent != curr_intent:
-        return True
-    # Action heading-like appears (e.g., 'Updation:', 'Deletion:')
-    if ":" in ln and any(k in low for k in ("updation", "update", "deletion", "delete", "cancel")):
-        return True
-    # Escalation marker
-    if any(m in low for m in ESCALATION_MARKERS):
-        return True
-    return False
-
-def synthetic_title(intent: str, action: str) -> str:
-    """Create a synthetic section title when headings are missing."""
-    base = {
-        "steps": "Process Steps",
-        "errors": "Common Errors & Resolution",
-        "prereqs": "Pre-Requisites",
-        "purpose": "Purpose",
-        "escalation": "Escalation Path",
-        "neutral": "Section"
-    }.get(intent, "Section")
-    if intent == "steps" and action:
-        return f"{base} — {action.capitalize()}"
-    return base
-
-def semantic_sectionize(paragraphs: List[str]) -> List[Tuple[str, List[str], Dict[str, str]]]:
-    """
-    Split the document into synthetic sections based on changing intent signals.
-    Returns: (section_title, lines, {'intent': ..., 'action': ...})
-    """
-    sections: List[Tuple[str, List[str], Dict[str, str]]] = []
-    current_block: List[str] = []
-    current_intent: Optional[str] = None
-
-    for ln in [p for p in paragraphs if (p or "").strip()]:
-        ln_intent = dominant_intent([ln])
-        block_intent = dominant_intent(current_block + [ln]) if current_block else ln_intent
-
-        if current_block and is_boundary_to_new_section(current_intent or "neutral", block_intent, ln):
-            # close current section
-            act = infer_action(current_block)
-            title = synthetic_title(current_intent or "neutral", act)
-            sections.append((title, current_block[:], {"intent": current_intent or "neutral", "action": act}))
-            # start new block
-            current_block = [ln]
-            current_intent = ln_intent
-        else:
-            current_block.append(ln)
-            current_intent = block_intent
-
-    # close last
-    if current_block:
-        act = infer_action(current_block)
-        title = synthetic_title(current_intent or "neutral", act)
-        sections.append((title, current_block[:], {"intent": current_intent or "neutral", "action": act}))
-
-    return sections
-
-# ----------------------------- Intent/module vocab used by runtime -----------------------------
+# ------------------------------ Intent & Module tagging ------------------------------
 SECTION_STEPS_HINTS = ["process steps", "procedure", "how to", "workflow", "instructions", "steps"]
-SECTION_ERRORS_HINTS = [
-    "common errors", "resolution", "troubleshooting", "known issues",
-    "common issues", "escalation", "permissions", "access"
-]
+SECTION_ERRORS_HINTS = ["common errors", "resolution", "troubleshooting", "known issues", "common issues", "escalation", "escalation path", "permissions", "access"]
 PERMISSION_TERMS = [
     "permission", "permissions", "access", "access right", "authorization", "authorisation",
     "role", "role access", "role mapping", "security", "security profile", "privilege", "insufficient",
     "not allowed", "not authorized", "denied", "restrict"
 ]
-ERROR_TERMS_R = ["error", "issue", "fail", "failure", "not working", "cannot", "can't", "mismatch", "locked", "wrong", "denied"]
-STEP_VERBS = [
-    "navigate", "select", "scan", "verify", "confirm", "print", "move", "complete", "click", "open",
-    "choose", "enter", "update", "save", "delete", "create", "attach", "assign", "reschedule", "edit", "change", "cancel", "remove", "tag"
-]
+ERROR_TERMS = ["error", "issue", "fail", "failure", "not working", "cannot", "can't", "mismatch", "locked", "wrong", "denied"]
+STEP_VERBS = ["navigate", "select", "scan", "verify", "confirm", "print", "move", "complete", "click", "open", "choose", "enter", "update", "save", "delete", "create", "attach", "assign"]
 MODULE_VOCAB = {
     "receiving": [
         "receive", "receiving", "inbound receiving", "inbound", "goods receipt", "grn",
@@ -259,8 +126,7 @@ MODULE_VOCAB = {
     ],
     "appointments": [
         "appointment", "appointments", "schedule", "scheduling", "slot", "dock door",
-        "appointment creation", "appointment details", "appointment schedule", "reschedule",
-        "updation", "update appointment", "cancel appointment", "delete appointment"
+        "appointment creation", "appointment details"
     ],
     "picking": ["pick", "picking", "pick release", "wave", "allocation"],
     "putaway": ["putaway", "staging", "put away", "location assignment"],
@@ -268,12 +134,22 @@ MODULE_VOCAB = {
     "inventory": ["inventory", "adjustment", "cycle count", "count", "uom"],
     "replenishment": ["replenishment", "replenish"],
 }
-ACTION_SYNONYMS = {
-    "create": ["create", "creation", "add", "new", "generate", "tag"],
-    "update": ["update", "modify", "change", "edit", "reschedule", "re-schedule", "updation"],
-    "delete": ["delete", "remove", "cancel", "deletion", "unassign"],
-    "navigate": ["navigate", "go to", "open"],
-}
+
+def _infer_intent_tag(section_title: str) -> str:
+    st = (section_title or "").lower()
+    if any(k in st for k in SECTION_STEPS_HINTS):
+        return "steps"
+    if any(k in st for k in SECTION_ERRORS_HINTS):
+        return "errors"
+    if "pre" in st and "requisite" in st:
+        return "prereqs"
+    if any(k in st for k in ["purpose", "overview", "introduction"]):
+        return "purpose"
+    if any(k in st for k in ["inbound receiving", "receiving", "goods receipt", "grn"]):
+        return "steps"
+    if any(k in st for k in ["appointment", "appointments", "schedule", "scheduling"]):
+        return "steps"
+    return "neutral"
 
 def _derive_semantic_intent_from_text(text: str) -> Tuple[str, List[str]]:
     t = (text or "").lower()
@@ -281,9 +157,9 @@ def _derive_semantic_intent_from_text(text: str) -> Tuple[str, List[str]]:
     intent = "neutral"
     if any(term in t for term in PERMISSION_TERMS):
         intent = "errors"; tags.append("permissions")
-    if "role" in t: tags.append("role_access")
-    if "security" in t: tags.append("security")
-    if intent == "neutral" and any(term in t for term in ERROR_TERMS_R):
+        if "role" in t: tags.append("role_access")
+        if "security" in t: tags.append("security")
+    if intent == "neutral" and any(term in t for term in ERROR_TERMS):
         intent = "errors"; tags.append("errors")
     if intent == "neutral" and any(v in t for v in STEP_VERBS):
         intent = "steps"; tags.append("procedure")
@@ -304,10 +180,10 @@ def _derive_module_tags(text: str, filename: str, section_title: str) -> List[st
             found = ["appointments"]
     return list(sorted(set(found)))
 
-# ----------------------------- Ingestion -----------------------------
+# ------------------------------ Ingestion ------------------------------
 def ingest_documents(folder_path: str) -> None:
     print(f"[KB] Checking folder: {folder_path}")
-    files = [f for f in os.listdir(folder_path) if f.lower().endswith(".docx")]
+    files = [f for f in os.listdir(folder_path) if f.lower().endswith('.docx')]
     print(f"[KB] Found {len(files)} Word files: {files}")
     if not files:
         print("[KB] WARNING: No .docx files found. Please check the folder path.")
@@ -321,23 +197,20 @@ def ingest_documents(folder_path: str) -> None:
         file_path = os.path.join(folder_path, file)
         doc_title = os.path.splitext(file)[0]
         doc = Document(file_path)
-        # Build raw paragraphs
-        paragraphs = [p.text.strip() for p in getattr(doc, "paragraphs", []) if p.text and p.text.strip()]
-        # Generic (heading-agnostic) sectionizer
-        sections = semantic_sectionize(paragraphs)
+        sections = _split_by_sections(doc)
         total_chunks = 0
-
-        for s_idx, (section_title, sec_lines, hints) in enumerate(sections):
-            chunks = _chunk_text_with_context(doc_title, section_title, sec_lines, max_words=160)
+        for s_idx, (section_title, paras) in enumerate(sections):
+            chunks = _chunk_text_with_context(doc_title, section_title, paras, max_words=160)
             total_chunks += len(chunks)
-            intent_tag_hint = hints.get("intent", "neutral")
-            action_tag_hint = hints.get("action", "")
-
+            base_intent = _infer_intent_tag(section_title)
             for c_idx, chunk in enumerate(chunks):
                 derived_intent, topic_tags = _derive_semantic_intent_from_text(chunk)
-                final_intent = intent_tag_hint if intent_tag_hint != "neutral" else derived_intent
+                final_intent = base_intent
+                if derived_intent == "errors":
+                    final_intent = "errors"
+                elif base_intent == "neutral" and derived_intent in ("steps", "prereqs"):
+                    final_intent = derived_intent
                 module_tags = _derive_module_tags(chunk, file, section_title)
-
                 embedding = model.encode(chunk).tolist()
                 doc_id = f"{file}:{s_idx}:{c_idx}"
                 meta = {
@@ -349,9 +222,7 @@ def ingest_documents(folder_path: str) -> None:
                     "intent_tag": final_intent,
                     "topic_tags": ", ".join(topic_tags) if topic_tags else "",
                     "module_tags": ", ".join(module_tags) if module_tags else "",
-                    "action_tag": action_tag_hint,
                 }
-
                 try:
                     collection.add(ids=[doc_id], embeddings=[embedding], documents=[chunk], metadatas=[meta])
                 except Exception:
@@ -361,7 +232,6 @@ def ingest_documents(folder_path: str) -> None:
                     except Exception as e2:
                         print(f"[KB] ERROR: Upsert failed for {doc_id}: {e2}")
 
-                # BM25 updates
                 tokens = _tokenize(chunk)
                 tf: Dict[str, int] = {}
                 for tkn in tokens:
@@ -375,15 +245,13 @@ def ingest_documents(folder_path: str) -> None:
                     "length": len(tokens),
                     "meta": meta,
                 })
-                seen_terms = set()
+                seen = set()
                 for term in tf.keys():
                     bm25_inverted.setdefault(term, []).append(idx)
-                    if term not in seen_terms:
+                    if term not in seen:
                         bm25_df[term] = bm25_df.get(term, 0) + 1
-                        seen_terms.add(term)
-
+                        seen.add(term)
         print(f"[KB] Ingested {file} → {total_chunks} chunks")
-
     N = len(bm25_docs)
     if N > 0:
         bm25_avgdl = sum(d["length"] for d in bm25_docs) / float(N)
@@ -402,7 +270,7 @@ def ingest_documents(folder_path: str) -> None:
     print(f"[KB] BM25 index saved: {BM25_INDEX_FILE}")
     print(f"[KB] Documents ingested. Total entries in Chroma: {collection.count()}")
 
-# ----------------------------- BM25 load -----------------------------
+# ------------------------------ BM25 load ------------------------------
 def _load_bm25_index() -> None:
     global bm25_docs, bm25_inverted, bm25_df, bm25_avgdl, bm25_ready
     if not os.path.exists(BM25_INDEX_FILE):
@@ -422,7 +290,7 @@ def _load_bm25_index() -> None:
 
 _load_bm25_index()
 
-# ----------------------------- BM25 search -----------------------------
+# ------------------------------ BM25 search ------------------------------
 def _bm25_score_for_doc(query_terms: List[str], doc_idx: int) -> float:
     if not bm25_ready or doc_idx < 0 or doc_idx >= len(bm25_docs):
         return 0.0
@@ -437,9 +305,10 @@ def _bm25_score_for_doc(query_terms: List[str], doc_idx: int) -> float:
         if tf == 0:
             continue
         N = len(bm25_docs)
+        idf_ratio = ((N - df + 0.5) / (df + 0.5))
         try:
             import math
-            idf = math.log(((N - df + 0.5) / (df + 0.5)) + 1.0)
+            idf = math.log(idf_ratio + 1.0)
         except Exception:
             idf = 1.0
         denom = tf + BM25_K1 * (1 - BM25_B + BM25_B * (dl / (bm25_avgdl or 1.0)))
@@ -467,17 +336,18 @@ def bm25_search(query: str, top_k: int = 50) -> List[Tuple[int, float]]:
     scored.sort(key=lambda x: x[1], reverse=True)
     return scored[:top_k]
 
-# ----------------------------- Semantic-only -----------------------------
+# ------------------------------ Semantic-only ------------------------------
 def search_knowledge_base(query: str, top_k: int = 10) -> dict:
     query_embedding = model.encode(query).tolist()
     res = collection.query(
         query_embeddings=[query_embedding],
         n_results=top_k,
-        include=['documents', 'metadatas', 'distances']
+        include=['documents', 'metadatas', 'distances']  # no 'ids'
     )
-    documents = (res.get("documents", [[""]]) or [[""]])[0]
-    metadatas = (res.get("metadatas", [[{}]]) or [[{}]])[0]
-    distances = (res.get("distances", [[None]]) or [[None]])[0]
+    documents = (res.get("documents", [[]]) or [[]])[0]
+    metadatas = (res.get("metadatas", [[]]) or [[]])[0]
+    distances = (res.get("distances", [[]]) or [[]])[0]
+    # Synthesize IDs from metadata (filename:section:chunk_index)
     ids: List[str] = []
     if documents:
         synthesized = []
@@ -495,15 +365,23 @@ def search_knowledge_base(query: str, top_k: int = 10) -> dict:
         "ids": ids,
     }
 
-# ----------------------------- Hybrid search -----------------------------
+# ------------------------------ Hybrid search (generic + intent-aware) ------------------------------
+ACTION_SYNONYMS = {
+    "create": ["create", "creation", "add", "new", "generate"],
+    "update": ["update", "modify", "change", "edit"],
+    "delete": ["delete", "remove"],
+    "navigate": ["navigate", "go to", "open"],
+}
+ERROR_INTENT_TERMS = [
+    "error", "issue", "fail", "not working", "resolution", "fix",
+    "permission", "permissions", "access", "no access", "authorization", "authorisation",
+    "role", "role mapping", "not authorized", "permission denied", "insufficient privileges",
+    "escalation", "escalation path", "access right", "mismatch", "locked", "wrong"
+]
+
 def _detect_user_intent(query: str) -> str:
     q = (query or "").lower()
-    if any(k in q for k in [
-        "error", "issue", "fail", "not working", "resolution", "fix",
-        "permission", "permissions", "access", "no access", "authorization", "authorisation",
-        "role", "role mapping", "not authorized", "permission denied", "insufficient privileges",
-        "escalation", "escalation path", "access right", "mismatch", "locked", "wrong"
-    ]):
+    if any(k in q for k in ERROR_INTENT_TERMS):
         return "errors"
     if any(k in q for k in ["steps", "procedure", "how to", "navigate", "process", "do", "perform"]):
         return "steps"
@@ -572,7 +450,7 @@ def _intent_weight(meta: dict, user_intent: str) -> float:
         or ("permissions" in topic_list)
     ):
         return 1.10
-    if user_intent == "steps" and any(k in st for k in ["process steps", "procedure", "instructions", "workflow", "creation", "updation", "deletion"]):
+    if user_intent == "steps" and any(k in st for k in ["process steps", "procedure", "instructions", "workflow"]):
         return 0.75
     return -0.2
 
@@ -582,8 +460,7 @@ def _meta_overlap(meta: Dict[str, Any], q_terms: List[str]) -> float:
     section_tokens = _tokenize_meta_value(meta.get("section"))
     topic_tokens = _tokenize_meta_value((meta.get("topic_tags") or ""))
     module_tokens = _tokenize_meta_value((meta.get("module_tags") or ""))
-    action_token = _tokenize_meta_value((meta.get("action_tag") or ""))
-    meta_tokens = set(fn_tokens + title_tokens + section_tokens + topic_tokens + module_tokens + action_token)
+    meta_tokens = set(fn_tokens + title_tokens + section_tokens + topic_tokens + module_tokens)
     if not meta_tokens or not q_terms:
         return 0.0
     qset = set(q_terms)
@@ -623,12 +500,17 @@ def _literal_query_match_boost(text: str, query_norm: str) -> float:
     return min(boost, 1.6)
 
 def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6, beta: float = 0.4) -> dict:
+    """
+    Hybrid retrieval (embeddings + BM25) with intent-, action-, module-, and phrase-aware reranking.
+    Returns top items plus doc-level prior and intent for downstream formatting.
+    """
     norm_query = _normalize_query(query)
     q_terms = _tokenize(norm_query)
     user_intent = _detect_user_intent(query)
     actions = _extract_actions(query)
     user_modules = _extract_modules_from_query(query)
 
+    # semantic (embeddings) search via Chroma
     sem_res = search_knowledge_base(norm_query, top_k=max(top_k, 40))
     sem_docs = sem_res.get("documents", [])
     sem_metas = sem_res.get("metadatas", [])
@@ -642,8 +524,9 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
             return 1.0 / (1.0 + float(d))
         except Exception:
             return 0.0
-
     sem_sims = [dist_to_sim(d) for d in sem_dists]
+
+    # BM25 search
     bm25_hits = bm25_search(norm_query, top_k=max(80, top_k * 6))
     bm25_max = max([s for _, s in bm25_hits], default=1.0)
     bm25_norm_pairs = [(idx, (score / bm25_max) if bm25_max > 0 else 0.0) for idx, score in bm25_hits]
@@ -654,23 +537,22 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
         bm25_id_to_text[d["id"]] = d["text"]
         bm25_id_to_meta[d["id"]] = d["meta"]
 
+    # union of candidate IDs (semantic + bm25)
     union_ids = set(sem_ids) | set(bm25_id_to_norm.keys())
-    gamma = 0.30
-    delta = 0.55
-    epsilon = 0.35
-    zeta = 0.65
-    eta = 0.50
-    iota = 0.60
-
-    def _action_meta_bonus(meta: Dict[str, Any], actions: List[str]) -> float:
-        if not actions:
-            return 0.0
-        act = (meta or {}).get("action_tag", "") or ""
-        return 0.6 if act and act in actions else 0.0
 
-    combined_records_ext: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float]] = []
+    # weights
+    gamma = 0.30  # meta overlap
+    delta = 0.55  # intent boost
+    epsilon = 0.30  # action weight
+    zeta = 0.65  # module weight
+    eta = 0.50  # phrase-level boost
+    theta = 0.00  # optional heading alignment bonus not used
+    iota = 0.60  # literal query match boost
+
+    combined_records_ext: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]] = []
 
     for cid in union_ids:
+        # pick semantic fields if present; fallback to bm25
         if cid in sem_ids:
             pos = sem_ids.index(cid)
             sem_sim = sem_sims[pos] if pos < len(sem_sims) else 0.0
@@ -690,7 +572,6 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
         m_overlap = _meta_overlap(meta, q_terms)
         intent_boost = _intent_weight(meta, user_intent)
         act_wt = _action_weight(text, actions)
-        act_meta = _action_meta_bonus(meta, actions)
         mod_wt = _module_weight(meta, user_modules)
         phrase_wt = _phrase_boost_score(text, q_terms)
         literal_wt = _literal_query_match_boost(text, norm_query)
@@ -700,48 +581,67 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
             + beta * bm25_sim
             + gamma * m_overlap
             + delta * intent_boost
-            + epsilon * (act_wt + act_meta)
+            + epsilon * act_wt
             + zeta * mod_wt
             + eta * phrase_wt
+            + theta * 0.0
             + iota * literal_wt
         )
         combined_records_ext.append(
             (cid, final_score, (sem_dist if sem_dist is not None else 999.0), text, meta,
-             m_overlap, intent_boost, act_wt + act_meta, mod_wt, phrase_wt, literal_wt)
+             m_overlap, intent_boost, act_wt, mod_wt, phrase_wt, 0.0, literal_wt)
         )
 
+    # exact-match rerank for errors (push lines containing query phrases)
+    if user_intent == "errors":
+        exact_hits = []
+        toks = [tok for tok in norm_query.split() if len(tok) > 2]
+        bigrams = _make_ngrams(toks, 2)
+        for rec in combined_records_ext:
+            text_lower = (rec[3] or "").lower()
+            if norm_query and norm_query in text_lower:
+                exact_hits.append(rec)
+                continue
+            if any(bg in text_lower for bg in bigrams):
+                exact_hits.append(rec)
+        if exact_hits:
+            rest = [r for r in combined_records_ext if r not in exact_hits]
+            exact_hits.sort(key=lambda x: x[1], reverse=True)
+            rest.sort(key=lambda x: x[1], reverse=True)
+            combined_records_ext = exact_hits + rest
+
+    # doc-level prior: prefer docs with more aligned chunks
     from collections import defaultdict as _dd
-    doc_groups: Dict[str, List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float]]] = _dd(list)
+    doc_groups: Dict[str, List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]]] = _dd(list)
     for rec in combined_records_ext:
         meta = rec[4] or {}
         fn = meta.get("filename", "unknown")
         doc_groups[fn].append(rec)
 
-    def doc_prior(recs: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float]]) -> float:
+    def doc_prior(recs: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]]) -> float:
         total_score = sum(r[1] for r in recs)
         total_overlap = sum(r[5] for r in recs)
         total_intent = sum(max(0.0, r[6]) for r in recs)
         total_action = sum(max(0.0, r[7]) for r in recs)
         total_module = sum(r[8] for r in recs)
         total_phrase = sum(r[9] for r in recs)
-        total_literal = sum(r[10] for r in recs)
+        total_literal = sum(r[11] for r in recs)
+        total_penalty = sum(min(0.0, r[6]) for r in recs) + sum(min(0.0, r[7]) for r in recs)
         errors_section_bonus = 0.0
-        if any(
-            "error" in ((r[4] or {}).get("section", "")).lower()
-            or "known issues" in ((r[4] or {}).get("section", "")).lower()
-            or "common issues" in ((r[4] or {}).get("section", "")).lower()
-            for r in recs
-        ):
+        if any("error" in ((r[4] or {}).get("section", "")).lower() or
+               "known issues" in ((r[4] or {}).get("section", "")).lower() or
+               "common issues" in ((r[4] or {}).get("section", "")).lower() for r in recs):
             errors_section_bonus = 0.5
         return (
             total_score
             + 0.4 * total_overlap
             + 0.7 * total_intent
-            + 0.55 * total_action
+            + 0.5 * total_action
             + 0.8 * total_module
             + 0.6 * total_phrase
             + 0.7 * total_literal
             + errors_section_bonus
+            + 0.3 * total_penalty
         )
 
     best_doc, best_doc_prior = None, -1.0
@@ -751,7 +651,7 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
             best_doc_prior, best_doc = p, fn
 
     best_recs = sorted(doc_groups.get(best_doc, []), key=lambda x: x[1], reverse=True)
-    other_recs: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float]] = []
+    other_recs: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]] = []
     for fn, recs in doc_groups.items():
         if fn == best_doc:
             continue
@@ -778,7 +678,7 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
         "actions": actions,
     }
 
-# ----------------------------- Section fetch helpers -----------------------------
+# ------------------------------ Section fetch helpers ------------------------------
 def get_section_text(filename: str, section: str) -> str:
     texts: List[str] = []
     for d in bm25_docs:
@@ -814,6 +714,7 @@ def get_best_errors_section_text(filename: str) -> str:
             or "access" in sec
             or "known issues" in sec
             or "common issues" in sec
+            or "errors" in sec
             or ("permissions" in topic_list)
         ):
             t = (d.get("text") or "").strip()
@@ -822,6 +723,10 @@ def get_best_errors_section_text(filename: str) -> str:
     return "\n\n".join(texts).strip()
 
 def get_escalation_text(filename: str) -> str:
+    """
+    Return concatenated text from any 'Escalation' section in the given SOP file.
+    Works across future SOPs—only relies on the heading name containing 'escalation'.
+    """
     texts: List[str] = []
     for d in bm25_docs:
         m = d.get("meta", {})
@@ -833,6 +738,7 @@ def get_escalation_text(filename: str) -> str:
                     texts.append(t)
     return "\n\n".join(texts).strip()
 
+# ------------------------------ Admin helpers ------------------------------
 def get_kb_runtime_info() -> Dict[str, Any]:
     return {
         "chroma_path": CHROMA_PATH,