Update services/kb_creation.py

services/kb_creation.py

@@ -1,4 +1,3 @@
-
 # services/kb_creation.py
 import os
 import re
@@ -8,12 +7,15 @@ from docx import Document
 from sentence_transformers import SentenceTransformer
 import chromadb
 
+# ---------------------------- ChromaDB setup ----------------------------
 CHROMA_PATH = os.path.join(os.getcwd(), "chroma_db")
 client = chromadb.PersistentClient(path=CHROMA_PATH)
 collection = client.get_or_create_collection(name="knowledge_base")
 
+# ---------------------------- Embedding model ----------------------------
 model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 
+# ---------------------------- BM25 (lightweight) ----------------------------
 BM25_INDEX_FILE = os.path.join(CHROMA_PATH, "bm25_index.pkl")
 bm25_docs: List[Dict[str, Any]] = []
 bm25_inverted: Dict[str, List[int]] = {}
@@ -23,6 +25,7 @@ bm25_ready: bool = False
 BM25_K1 = 1.5
 BM25_B = 0.75
 
+# ---------------------------- Utilities ----------------------------
 def _tokenize(text: str) -> List[str]:
     if not text:
         return []
@@ -38,7 +41,8 @@ def _normalize_query(q: str) -> str:
 def _tokenize_meta_value(val: Optional[str]) -> List[str]:
     return _tokenize(val or "")
 
-BULLET_RE = re.compile(r"^\s*(?:[\-\*•]|\d+[.)])\s+", re.IGNORECASE)
+# ---------------------------- DOCX parsing & chunking ----------------------------
+BULLET_RE = re.compile(r"^\s*(?:[\-\*\u2022]|\d+[.)])\s+", re.IGNORECASE)
 
 def _split_by_sections(doc: Document) -> List[Tuple[str, List[str]]]:
     sections: List[Tuple[str, List[str]]] = []
@@ -64,6 +68,11 @@ def _split_by_sections(doc: Document) -> List[Tuple[str, List[str]]]:
     return sections
 
 def _paragraphs_to_lines(paragraphs: List[str]) -> List[str]:
+    """
+    Split paragraphs into bullet-aware lines:
+    - Preserve bullets/numbered list lines as separate atomic lines.
+    - Split long paragraphs by sentence boundaries.
+    """
     lines: List[str] = []
     for p in (paragraphs or []):
         p = (p or "").strip()
@@ -72,15 +81,19 @@ def _paragraphs_to_lines(paragraphs: List[str]) -> List[str]:
         if BULLET_RE.match(p):
             lines.append(p)
             continue
-        parts = [s.strip() for s in re.split(r"(?<=[.!?])\s+|;\s+|\s*→\s*|\s*->\s*|\s*then\s*", p) if s.strip()]
+        parts = [s.strip() for s in re.split(r"(?<=[.!?])\s+", p) if s.strip()]
         lines.extend(parts)
     return lines
 
 def _chunk_text_with_context(doc_title: str, section_title: str, paragraphs: List[str], max_words: int = 160) -> List[str]:
+    """
+    Smaller chunks (≈160 words), bullet-aware for better recall of error bullets.
+    """
     lines = _paragraphs_to_lines(paragraphs)
     chunks: List[str] = []
     current: List[str] = []
     current_len = 0
+
     for ln in lines:
         w = ln.split()
         if current_len + len(w) > max_words or (BULLET_RE.match(ln) and current):
@@ -92,18 +105,22 @@ def _chunk_text_with_context(doc_title: str, section_title: str, paragraphs: Lis
         else:
             current.append(ln)
             current_len += len(w)
+
     if current:
         chunk = " ".join(current).strip()
         if chunk:
             chunks.append(chunk)
+
     if not chunks:
         body = " ".join(lines).strip()
         if body:
             chunks = [body]
     return chunks
 
+# ---------------------------- Intent & Module tagging ----------------------------
 SECTION_STEPS_HINTS = ["process steps", "procedure", "how to", "workflow", "instructions", "steps"]
 SECTION_ERRORS_HINTS = ["common errors", "resolution", "troubleshooting", "known issues", "common issues", "escalation", "escalation path", "permissions", "access"]
+
 PERMISSION_TERMS = [
     "permission", "permissions", "access", "access right", "authorization", "authorisation",
     "role", "role access", "role mapping", "security", "security profile", "privilege", "insufficient",
@@ -111,9 +128,16 @@ PERMISSION_TERMS = [
 ]
 ERROR_TERMS = ["error", "issue", "fail", "failure", "not working", "cannot", "can't", "mismatch", "locked", "wrong", "denied"]
 STEP_VERBS = ["navigate", "select", "scan", "verify", "confirm", "print", "move", "complete", "click", "open", "choose", "enter", "update", "save", "delete", "create", "attach", "assign"]
+
 MODULE_VOCAB = {
-    "receiving": ["receive", "receiving", "inbound receiving", "inbound", "goods receipt", "grn", "asn receiving", "unload", "check-in", "dock check-in"],
-    "appointments": ["appointment", "appointments", "schedule", "scheduling", "slot", "dock door", "appointment creation", "appointment details"],
+    "receiving": [
+        "receive", "receiving", "inbound receiving", "inbound", "goods receipt", "grn",
+        "asn receiving", "unload", "check-in", "dock check-in"
+    ],
+    "appointments": [
+        "appointment", "appointments", "schedule", "scheduling", "slot", "dock door",
+        "appointment creation", "appointment details"
+    ],
     "picking": ["pick", "picking", "pick release", "wave", "allocation"],
     "putaway": ["putaway", "staging", "put away", "location assignment"],
     "shipping": ["shipping", "ship confirm", "outbound", "load", "trailer"],
@@ -171,24 +195,7 @@ def _derive_module_tags(text: str, filename: str, section_title: str) -> List[st
             found = ["appointments"]
     return list(sorted(set(found)))
 
-ACTION_SYNONYMS = {
-    "create": ["create", "creation", "add", "new", "generate", "book", "schedule"],
-    "update": ["update", "modify", "change", "edit", "amend", "reschedule", "re-schedule"],
-    "delete": ["delete", "remove"],
-    "navigate": ["navigate", "go to", "open"],
-}
-
-def _derive_action_tags(text: str) -> List[str]:
-    t = (text or "").lower()
-    tags = []
-    for act, syns in ACTION_SYNONYMS.items():
-        if any(s in t for s in syns):
-            tags.append(act)
-    if "reschedule" in t or "re-schedule" in t:
-        if "update" not in tags:
-            tags.append("update")
-    return sorted(set(tags))
-
+# ---------------------------- Ingestion ----------------------------
 def ingest_documents(folder_path: str) -> None:
     print(f"[KB] Checking folder: {folder_path}")
     files = [f for f in os.listdir(folder_path) if f.lower().endswith('.docx')]
@@ -196,19 +203,24 @@ def ingest_documents(folder_path: str) -> None:
     if not files:
         print("[KB] WARNING: No .docx files found. Please check the folder path.")
         return
+
     global bm25_docs, bm25_inverted, bm25_df, bm25_avgdl, bm25_ready
     bm25_docs, bm25_inverted, bm25_df = [], {}, {}
     bm25_avgdl, bm25_ready = 0.0, False
+
     for file in files:
         file_path = os.path.join(folder_path, file)
         doc_title = os.path.splitext(file)[0]
         doc = Document(file_path)
         sections = _split_by_sections(doc)
         total_chunks = 0
+
         for s_idx, (section_title, paras) in enumerate(sections):
             chunks = _chunk_text_with_context(doc_title, section_title, paras, max_words=160)
             total_chunks += len(chunks)
+
             base_intent = _infer_intent_tag(section_title)
+
             for c_idx, chunk in enumerate(chunks):
                 derived_intent, topic_tags = _derive_semantic_intent_from_text(chunk)
                 final_intent = base_intent
@@ -216,8 +228,8 @@ def ingest_documents(folder_path: str) -> None:
                     final_intent = "errors"
                 elif base_intent == "neutral" and derived_intent in ("steps", "prereqs"):
                     final_intent = derived_intent
+
                 module_tags = _derive_module_tags(chunk, file, section_title)
-                action_tags = _derive_action_tags(chunk)
                 embedding = model.encode(chunk).tolist()
                 doc_id = f"{file}:{s_idx}:{c_idx}"
                 meta = {
@@ -229,7 +241,6 @@ def ingest_documents(folder_path: str) -> None:
                     "intent_tag": final_intent,
                     "topic_tags": ", ".join(topic_tags) if topic_tags else "",
                     "module_tags": ", ".join(module_tags) if module_tags else "",
-                    "action_tags": ", ".join(action_tags) if action_tags else "",
                 }
                 try:
                     collection.add(ids=[doc_id], embeddings=[embedding], documents=[chunk], metadatas=[meta])
@@ -239,10 +250,12 @@ def ingest_documents(folder_path: str) -> None:
                         collection.add(ids=[doc_id], embeddings=[embedding], documents=[chunk], metadatas=[meta])
                     except Exception as e2:
                         print(f"[KB] ERROR: Upsert failed for {doc_id}: {e2}")
+
                 tokens = _tokenize(chunk)
                 tf: Dict[str, int] = {}
                 for tkn in tokens:
                     tf[tkn] = tf.get(tkn, 0) + 1
+
                 idx = len(bm25_docs)
                 bm25_docs.append({
                     "id": doc_id,
@@ -258,11 +271,14 @@ def ingest_documents(folder_path: str) -> None:
                     if term not in seen:
                         bm25_df[term] = bm25_df.get(term, 0) + 1
                         seen.add(term)
+
         print(f"[KB] Ingested {file} → {total_chunks} chunks")
+
     N = len(bm25_docs)
     if N > 0:
         bm25_avgdl = sum(d["length"] for d in bm25_docs) / float(N)
         bm25_ready = True
+
     payload = {
         "bm25_docs": bm25_docs,
         "bm25_inverted": bm25_inverted,
@@ -277,7 +293,7 @@ def ingest_documents(folder_path: str) -> None:
     print(f"[KB] BM25 index saved: {BM25_INDEX_FILE}")
     print(f"[KB] Documents ingested. Total entries in Chroma: {collection.count()}")
 
-
+# ---------------------------- BM25 load ----------------------------
 def _load_bm25_index() -> None:
     global bm25_docs, bm25_inverted, bm25_df, bm25_avgdl, bm25_ready
     if not os.path.exists(BM25_INDEX_FILE):
@@ -297,7 +313,7 @@ def _load_bm25_index() -> None:
 
 _load_bm25_index()
 
-
+# ---------------------------- BM25 search ----------------------------
 def _bm25_score_for_doc(query_terms: List[str], doc_idx: int) -> float:
     if not bm25_ready or doc_idx < 0 or doc_idx >= len(bm25_docs):
         return 0.0
@@ -312,9 +328,10 @@ def _bm25_score_for_doc(query_terms: List[str], doc_idx: int) -> float:
         if tf == 0:
             continue
         N = len(bm25_docs)
+        idf_ratio = ((N - df + 0.5) / (df + 0.5))
         try:
             import math
-            idf = math.log(((N - df + 0.5) / (df + 0.5)) + 1.0)
+            idf = math.log(idf_ratio + 1.0)
         except Exception:
             idf = 1.0
         denom = tf + BM25_K1 * (1 - BM25_B + BM25_B * (dl / (bm25_avgdl or 1.0)))
@@ -342,13 +359,19 @@ def bm25_search(query: str, top_k: int = 50) -> List[Tuple[int, float]]:
     scored.sort(key=lambda x: x[1], reverse=True)
     return scored[:top_k]
 
-
+# ---------------------------- Semantic-only ----------------------------
 def search_knowledge_base(query: str, top_k: int = 10) -> dict:
     query_embedding = model.encode(query).tolist()
-    res = collection.query(query_embeddings=[query_embedding], n_results=top_k, include=['documents', 'metadatas', 'distances'])
+    res = collection.query(
+        query_embeddings=[query_embedding],
+        n_results=top_k,
+        include=['documents', 'metadatas', 'distances']  # no 'ids'
+    )
     documents = (res.get("documents", [[]]) or [[]])[0]
     metadatas = (res.get("metadatas", [[]]) or [[]])[0]
     distances = (res.get("distances", [[]]) or [[]])[0]
+
+    # Synthesize IDs from metadata (filename:section:chunk_index)
     ids: List[str] = []
     if documents:
         synthesized = []
@@ -358,8 +381,22 @@ def search_knowledge_base(query: str, top_k: int = 10) -> dict:
             idx = (m or {}).get("chunk_index", i)
             synthesized.append(f"{fn}:{sec}:{idx}")
         ids = synthesized
-    return {"documents": documents, "metadatas": metadatas, "distances": distances, "ids": ids}
 
+    print(f"[KB] search → {len(documents)} docs (top_k={top_k}); first distance: {distances[0] if distances else 'n/a'}; ids synthesized={len(ids)}")
+    return {
+        "documents": documents,
+        "metadatas": metadatas,
+        "distances": distances,
+        "ids": ids,
+    }
+
+# ---------------------------- Hybrid search (improved + exact-match rerank) ----------------------------
+ACTION_SYNONYMS = {
+    "create": ["create", "creation", "add", "new", "generate"],
+    "update": ["update", "modify", "change", "edit"],
+    "delete": ["delete", "remove"],
+    "navigate": ["navigate", "go to", "open"],
+}
 ERROR_INTENT_TERMS = [
     "error", "issue", "fail", "not working", "resolution", "fix",
     "permission", "permissions", "access", "no access", "authorization", "authorisation",
@@ -410,12 +447,12 @@ def _action_weight(text: str, actions: List[str]) -> float:
         for syn in ACTION_SYNONYMS.get(act, [act]):
             if syn in t:
                 score += 1.0
-    conflicts = {"create": ["update"], "update": ["create"], "delete": ["create", "update"], "navigate": []}
+    conflicts = {"create": ["delete"], "delete": ["create"], "update": ["delete"], "navigate": []}
     for act in actions:
         for bad in conflicts.get(act, []):
             for syn in ACTION_SYNONYMS.get(bad, [bad]):
                 if syn in t:
-                    score -= 1.0
+                    score -= 0.8
     return score
 
 def _module_weight(meta: Dict[str, Any], user_modules: List[str]) -> float:
@@ -439,11 +476,12 @@ def _intent_weight(meta: dict, user_intent: str) -> float:
     st = ((meta or {}).get("section", "") or "").lower()
     topics = (meta or {}).get("topic_tags", "") or ""
     topic_list = [t.strip() for t in topics.split(",") if t.strip()]
+    # Prefer errors sections strongly
     if user_intent == "errors" and (
-        any(k in st for k in ["common errors", "known issues", "common issues", "errors", "escalation", "permissions", "access"]) or
-        ("permissions" in topic_list)
+        any(k in st for k in ["common errors", "known issues", "common issues", "errors", "escalation", "permissions", "access"])
+        or ("permissions" in topic_list)
     ):
-        return 1.10
+        return 1.10  # stronger than before
     if user_intent == "steps" and any(k in st for k in ["inbound receiving", "receiving", "goods receipt", "grn"]):
         return 0.75
     return -0.2
@@ -480,6 +518,7 @@ def _phrase_boost_score(text: str, q_terms: List[str]) -> float:
     return min(score, 2.0)
 
 def _literal_query_match_boost(text: str, query_norm: str) -> float:
+    """Extra boost if exact normalized query substring or bigrams appear."""
     t = (text or "").lower()
     q = (query_norm or "").lower()
     boost = 0.0
@@ -493,22 +532,6 @@ def _literal_query_match_boost(text: str, query_norm: str) -> float:
             break
     return min(boost, 1.6)
 
-def _action_meta_weight(meta: Dict[str, Any], user_actions: List[str]) -> float:
-    raw = (meta or {}).get("action_tags", "") or ""
-    doc_actions = [a.strip().lower() for a in raw.split(",") if a.strip()] if isinstance(raw, str) else (raw or [])
-    if not user_actions:
-        return 0.0
-    ua = set(a.lower() for a in user_actions)
-    da = set(doc_actions)
-    overlap = len(ua & da)
-    if overlap > 0:
-        return 1.2 * overlap
-    if "update" in ua and "create" in da and "update" not in da:
-        return -1.4
-    if "create" in ua and "update" in da and "create" not in da:
-        return -1.2
-    return -0.4
-
 def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6, beta: float = 0.4) -> dict:
     norm_query = _normalize_query(query)
     q_terms = _tokenize(norm_query)
@@ -529,12 +552,12 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
             return 1.0 / (1.0 + float(d))
         except Exception:
             return 0.0
+
     sem_sims = [dist_to_sim(d) for d in sem_dists]
 
     bm25_hits = bm25_search(norm_query, top_k=max(80, top_k * 6))
     bm25_max = max([s for _, s in bm25_hits], default=1.0)
     bm25_norm_pairs = [(idx, (score / bm25_max) if bm25_max > 0 else 0.0) for idx, score in bm25_hits]
-
     bm25_id_to_norm, bm25_id_to_text, bm25_id_to_meta = {}, {}, {}
     for idx, nscore in bm25_norm_pairs:
         d = bm25_docs[idx]
@@ -544,17 +567,15 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
 
     union_ids = set(sem_ids) | set(bm25_id_to_norm.keys())
 
-    gamma = 0.30
-    delta = 0.55
-    epsilon = 0.50
-    zeta = 0.65
-    eta = 0.50
-    theta = 0.40
-    iota = 0.60
-    kappa = 0.90
-
-    combined_records_ext: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float, float]] = []
+    gamma = 0.30   # meta overlap
+    delta = 0.55   # intent boost (stronger)
+    epsilon = 0.30 # action weight
+    zeta = 0.65    # module weight
+    eta = 0.50     # phrase-level boost (stronger)
+    theta = 0.40   # heading alignment bonus
+    iota = 0.60    # literal query match boost (stronger)
 
+    combined_records_ext: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]] = []
     for cid in union_ids:
         if cid in sem_ids:
             pos = sem_ids.index(cid)
@@ -564,17 +585,21 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
             sem_meta = sem_metas[pos] if pos < len(sem_metas) else {}
         else:
             sem_sim, sem_dist, sem_text, sem_meta = 0.0, None, "", {}
+
         bm25_sim = bm25_id_to_norm.get(cid, 0.0)
         bm25_text = bm25_id_to_text.get(cid, "")
         bm25_meta = bm25_id_to_meta.get(cid, {})
+
         text = sem_text if sem_text else bm25_text
         meta = sem_meta if sem_meta else bm25_meta
+
         m_overlap = _meta_overlap(meta, q_terms)
         intent_boost = _intent_weight(meta, user_intent)
         act_wt = _action_weight(text, actions)
         mod_wt = _module_weight(meta, user_modules)
         phrase_wt = _phrase_boost_score(text, q_terms)
         literal_wt = _literal_query_match_boost(text, norm_query)
+
         sec_low = ((meta or {}).get("section", "") or "").lower()
         title_low = ((meta or {}).get("title", "") or "").lower()
         heading_bonus = 0.0
@@ -584,32 +609,49 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
             heading_bonus += 0.40
         if any(root in sec_low for root in ["appointment", "appointments", "schedule"]) and "receiv" in norm_query:
             heading_bonus -= 0.35
-        action_meta_wt = _action_meta_weight(meta, actions)
+
         final_score = (
-            alpha * sem_sim + beta * bm25_sim + gamma * m_overlap + delta * intent_boost + epsilon * act_wt + zeta * mod_wt + eta * phrase_wt + theta * heading_bonus + iota * literal_wt + kappa * action_meta_wt
+            alpha * sem_sim
+            + beta * bm25_sim
+            + gamma * m_overlap
+            + delta * intent_boost
+            + epsilon * act_wt
+            + zeta * mod_wt
+            + eta * phrase_wt
+            + theta * heading_bonus
+            + iota * literal_wt
+        )
+
+        combined_records_ext.append(
+            (cid, final_score, (sem_dist if sem_dist is not None else 999.0), text, meta, m_overlap, intent_boost, act_wt, mod_wt, phrase_wt, heading_bonus, literal_wt)
         )
-        combined_records_ext.append((cid, final_score, (sem_dist if sem_dist is not None else 999.0), text, meta, m_overlap, intent_boost, act_wt, mod_wt, phrase_wt, heading_bonus, literal_wt, action_meta_wt))
 
-    if _detect_user_intent(query) == "errors":
+    # ---- Exact-match rerank for errors ----
+    if user_intent == "errors":
         exact_hits = []
         for rec in combined_records_ext:
             text_lower = (rec[3] or "").lower()
-            if any(phrase in text_lower for phrase in [norm_query, *(_make_ngrams([tok for tok in norm_query.split() if len(tok) > 2], 2))]):
+            if any(phrase in text_lower for phrase in [
+                norm_query,  # whole normalized query
+                # common 2-gram patterns extracted from the query
+                *(_make_ngrams([tok for tok in norm_query.split() if len(tok) > 2], 2))
+            ]):
                 exact_hits.append(rec)
         if exact_hits:
+            # Move exact hits to front and keep order by current final_score
             rest = [r for r in combined_records_ext if r not in exact_hits]
             exact_hits.sort(key=lambda x: x[1], reverse=True)
             rest.sort(key=lambda x: x[1], reverse=True)
             combined_records_ext = exact_hits + rest
 
     from collections import defaultdict
-    doc_groups: Dict[str, List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float, float]]] = defaultdict(list)
+    doc_groups: Dict[str, List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]]] = defaultdict(list)
     for rec in combined_records_ext:
         meta = rec[4] or {}
         fn = meta.get("filename", "unknown")
         doc_groups[fn].append(rec)
 
-    def doc_prior(recs: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float, float]]) -> float:
+    def doc_prior(recs: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]]) -> float:
         total_score = sum(r[1] for r in recs)
         total_overlap = sum(r[5] for r in recs)
         total_intent = sum(max(0.0, r[6]) for r in recs)
@@ -618,13 +660,23 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
         total_phrase = sum(r[9] for r in recs)
         total_heading = sum(r[10] for r in recs)
         total_literal = sum(r[11] for r in recs)
-        total_action_meta = sum(r[12] for r in recs)
         total_penalty = sum(min(0.0, r[6]) for r in recs) + sum(min(0.0, r[7]) for r in recs)
+        # Errors doc prior: if many chunks are from an errors/known issues section, add a bonus
         errors_section_bonus = 0.0
-        if any("error" in ((r[4] or {}).get("section", "")).lower() or "known issues" in ((r[4] or {}).get("section", "")).lower() or "common issues" in ((r[4] or {}).get("section", "")).lower() for r in recs):
+        if any("error" in ((r[4] or {}).get("section", "")).lower() or "known issues" in ((r[4] or {}).get("section", "")).lower()
+               or "common issues" in ((r[4] or {}).get("section", "")).lower() for r in recs):
             errors_section_bonus = 0.5
         return (
-            total_score + 0.4 * total_overlap + 0.7 * total_intent + 0.5 * total_action + 0.8 * total_module + 0.6 * total_phrase + 0.6 * total_heading + 0.7 * total_literal + 0.9 * total_action_meta + errors_section_bonus + 0.3 * total_penalty
+            total_score
+            + 0.4 * total_overlap
+            + 0.7 * total_intent
+            + 0.5 * total_action
+            + 0.8 * total_module
+            + 0.6 * total_phrase
+            + 0.6 * total_heading
+            + 0.7 * total_literal
+            + errors_section_bonus
+            + 0.3 * total_penalty
         )
 
     best_doc, best_doc_prior = None, -1.0
@@ -634,22 +686,35 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
             best_doc_prior, best_doc = p, fn
 
     best_recs = sorted(doc_groups.get(best_doc, []), key=lambda x: x[1], reverse=True)
-    other_recs: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float, float]] = []
+    other_recs: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]] = []
     for fn, recs in doc_groups.items():
         if fn == best_doc:
             continue
         other_recs.extend(recs)
     other_recs.sort(key=lambda x: x[1], reverse=True)
+
     reordered = best_recs + other_recs
     top = reordered[:top_k]
+
     documents = [t[3] for t in top]
     metadatas = [t[4] for t in top]
     distances = [t[2] for t in top]
     ids = [t[0] for t in top]
     combined_scores = [t[1] for t in top]
-    return {"documents": documents, "metadatas": metadatas, "distances": distances, "ids": ids, "combined_scores": combined_scores, "best_doc": best_doc, "best_doc_prior": best_doc_prior, "user_intent": user_intent, "actions": actions}
 
+    return {
+        "documents": documents,
+        "metadatas": metadatas,
+        "distances": distances,
+        "ids": ids,
+        "combined_scores": combined_scores,
+        "best_doc": best_doc,
+        "best_doc_prior": best_doc_prior,
+        "user_intent": user_intent,
+        "actions": actions,
+    }
 
+# ---------------------------- Section fetch helpers ----------------------------
 def get_section_text(filename: str, section: str) -> str:
     texts: List[str] = []
     for d in bm25_docs:
@@ -658,7 +723,7 @@ def get_section_text(filename: str, section: str) -> str:
             t = (d.get("text") or "").strip()
             if t:
                 texts.append(t)
-    return "".join(texts).strip()
+    return "\n\n".join(texts).strip()
 
 def get_best_steps_section_text(filename: str) -> str:
     texts: List[str] = []
@@ -668,13 +733,13 @@ def get_best_steps_section_text(filename: str) -> str:
             t = (d.get("text") or "").strip()
             if t:
                 texts.append(t)
-    return "".join(texts).strip()
+    return "\n\n".join(texts).strip()
 
 def get_best_errors_section_text(filename: str) -> str:
     texts: List[str] = []
     for d in bm25_docs:
         m = d.get("meta", {})
-        sec = ((m.get("section") or "").lower())
+        sec = (m.get("section") or "").lower()
         topics = (m.get("topic_tags") or "")
         topic_list = [t.strip() for t in topics.split(",") if t.strip()]
         if m.get("filename") == filename and (
@@ -691,26 +756,9 @@ def get_best_errors_section_text(filename: str) -> str:
             t = (d.get("text") or "").strip()
             if t:
                 texts.append(t)
-    return "".join(texts).strip()
-
-def get_steps_text_by_action(filename: str, preferred_actions: List[str]) -> str:
-    if not preferred_actions:
-        return ""
-    actions = set(a.lower() for a in preferred_actions)
-    texts: List[str] = []
-    for d in bm25_docs:
-        m = d.get("meta", {})
-        if m.get("filename") != filename or m.get("intent_tag") != "steps":
-            continue
-        raw = (m.get("action_tags") or "").lower()
-        doc_actions = set(a.strip() for a in raw.split(",") if a.strip())
-        if actions & doc_actions:
-            t = (d.get("text") or "").strip()
-            if t:
-                texts.append(t)
-    return "".join(texts).strip()
-
+    return "\n\n".join(texts).strip()
 
+# ---------------------------- Admin helpers ----------------------------
 def get_kb_runtime_info() -> Dict[str, Any]:
     return {
         "chroma_path": CHROMA_PATH,
@@ -730,11 +778,13 @@ def reset_kb(folder_path: str) -> Dict[str, Any]:
             pass
         global collection
         collection = client.get_or_create_collection(name="knowledge_base")
+
         try:
             if os.path.isfile(BM25_INDEX_FILE):
                 os.remove(BM25_INDEX_FILE)
         except Exception as e:
             result.setdefault("warnings", []).append(f"bm25 index delete: {e}")
+
         os.makedirs(CHROMA_PATH, exist_ok=True)
         ingest_documents(folder_path)
         result["info"] = get_kb_runtime_info()