Update services/kb_creation.py

services/kb_creation.py

@@ -8,15 +8,15 @@ from docx import Document
 from sentence_transformers import SentenceTransformer
 import chromadb
 
-# ---------------------------- ChromaDB setup ----------------------------
+# --------------------------- ChromaDB setup ---------------------------
 CHROMA_PATH = os.path.join(os.getcwd(), "chroma_db")
 client = chromadb.PersistentClient(path=CHROMA_PATH)
 collection = client.get_or_create_collection(name="knowledge_base")
 
-# ---------------------------- Embedding model ----------------------------
+# --------------------------- Embedding model ---------------------------
 model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 
-# ---------------------------- BM25 (lightweight) ----------------------------
+# --------------------------- BM25 (lightweight) ---------------------------
 BM25_INDEX_FILE = os.path.join(CHROMA_PATH, "bm25_index.pkl")
 bm25_docs: List[Dict[str, Any]] = []
 bm25_inverted: Dict[str, List[int]] = {}
@@ -26,7 +26,7 @@ bm25_ready: bool = False
 BM25_K1 = 1.5
 BM25_B = 0.75
 
-# ---------------------------- Utilities ----------------------------
+# --------------------------- Utilities ---------------------------
 def _tokenize(text: str) -> List[str]:
     if not text:
         return []
@@ -42,8 +42,8 @@ def _normalize_query(q: str) -> str:
 def _tokenize_meta_value(val: Optional[str]) -> List[str]:
     return _tokenize(val or "")
 
-# ---------------------------- DOCX parsing & chunking ----------------------------
-BULLET_RE = re.compile(r"^\s*(?:[\-\*\u2022]|\d+[.)])\s+", re.IGNORECASE)
+# --------------------------- DOCX parsing & chunking ---------------------------
+BULLET_RE = re.compile(r"^\s*(?:[\-\*\u2022]|\d+[.\)]\s+)", re.IGNORECASE)
 
 def _split_by_sections(doc: Document) -> List[Tuple[str, List[str]]]:
     sections: List[Tuple[str, List[str]]] = []
@@ -87,14 +87,11 @@ def _paragraphs_to_lines(paragraphs: List[str]) -> List[str]:
     return lines
 
 def _chunk_text_with_context(doc_title: str, section_title: str, paragraphs: List[str], max_words: int = 160) -> List[str]:
-    """
-    Smaller chunks (≈160 words), bullet-aware for better recall of error bullets.
-    """
+    """Smaller chunks (~160 words), bullet-aware for better recall."""
     lines = _paragraphs_to_lines(paragraphs)
     chunks: List[str] = []
     current: List[str] = []
     current_len = 0
-
     for ln in lines:
         w = ln.split()
         if current_len + len(w) > max_words or (BULLET_RE.match(ln) and current):
@@ -106,22 +103,19 @@ def _chunk_text_with_context(doc_title: str, section_title: str, paragraphs: Lis
         else:
             current.append(ln)
             current_len += len(w)
-
     if current:
         chunk = " ".join(current).strip()
         if chunk:
             chunks.append(chunk)
-
     if not chunks:
         body = " ".join(lines).strip()
         if body:
             chunks = [body]
     return chunks
 
-# ---------------------------- Intent & Module tagging ----------------------------
+# --------------------------- Intent & Module tagging ---------------------------
 SECTION_STEPS_HINTS = ["process steps", "procedure", "how to", "workflow", "instructions", "steps"]
 SECTION_ERRORS_HINTS = ["common errors", "resolution", "troubleshooting", "known issues", "common issues", "escalation", "escalation path", "permissions", "access"]
-
 PERMISSION_TERMS = [
     "permission", "permissions", "access", "access right", "authorization", "authorisation",
     "role", "role access", "role mapping", "security", "security profile", "privilege", "insufficient",
@@ -129,7 +123,6 @@ PERMISSION_TERMS = [
 ]
 ERROR_TERMS = ["error", "issue", "fail", "failure", "not working", "cannot", "can't", "mismatch", "locked", "wrong", "denied"]
 STEP_VERBS = ["navigate", "select", "scan", "verify", "confirm", "print", "move", "complete", "click", "open", "choose", "enter", "update", "save", "delete", "create", "attach", "assign"]
-
 MODULE_VOCAB = {
     "receiving": [
         "receive", "receiving", "inbound receiving", "inbound", "goods receipt", "grn",
@@ -196,7 +189,7 @@ def _derive_module_tags(text: str, filename: str, section_title: str) -> List[st
             found = ["appointments"]
     return list(sorted(set(found)))
 
-# ---------------------------- Ingestion ----------------------------
+# --------------------------- Ingestion ---------------------------
 def ingest_documents(folder_path: str) -> None:
     print(f"[KB] Checking folder: {folder_path}")
     files = [f for f in os.listdir(folder_path) if f.lower().endswith('.docx')]
@@ -219,7 +212,6 @@ def ingest_documents(folder_path: str) -> None:
         for s_idx, (section_title, paras) in enumerate(sections):
             chunks = _chunk_text_with_context(doc_title, section_title, paras, max_words=160)
             total_chunks += len(chunks)
-
             base_intent = _infer_intent_tag(section_title)
 
             for c_idx, chunk in enumerate(chunks):
@@ -243,6 +235,7 @@ def ingest_documents(folder_path: str) -> None:
                     "topic_tags": ", ".join(topic_tags) if topic_tags else "",
                     "module_tags": ", ".join(module_tags) if module_tags else "",
                 }
+
                 try:
                     collection.add(ids=[doc_id], embeddings=[embedding], documents=[chunk], metadatas=[meta])
                 except Exception:
@@ -266,6 +259,7 @@ def ingest_documents(folder_path: str) -> None:
                     "length": len(tokens),
                     "meta": meta,
                 })
+
                 seen = set()
                 for term in tf.keys():
                     bm25_inverted.setdefault(term, []).append(idx)
@@ -279,7 +273,6 @@ def ingest_documents(folder_path: str) -> None:
     if N > 0:
         bm25_avgdl = sum(d["length"] for d in bm25_docs) / float(N)
         bm25_ready = True
-
     payload = {
         "bm25_docs": bm25_docs,
         "bm25_inverted": bm25_inverted,
@@ -294,7 +287,7 @@ def ingest_documents(folder_path: str) -> None:
     print(f"[KB] BM25 index saved: {BM25_INDEX_FILE}")
     print(f"[KB] Documents ingested. Total entries in Chroma: {collection.count()}")
 
-# ---------------------------- BM25 load ----------------------------
+# --------------------------- BM25 load ---------------------------
 def _load_bm25_index() -> None:
     global bm25_docs, bm25_inverted, bm25_df, bm25_avgdl, bm25_ready
     if not os.path.exists(BM25_INDEX_FILE):
@@ -314,7 +307,7 @@ def _load_bm25_index() -> None:
 
 _load_bm25_index()
 
-# ---------------------------- BM25 search ----------------------------
+# --------------------------- BM25 search ---------------------------
 def _bm25_score_for_doc(query_terms: List[str], doc_idx: int) -> float:
     if not bm25_ready or doc_idx < 0 or doc_idx >= len(bm25_docs):
         return 0.0
@@ -346,12 +339,14 @@ def bm25_search(query: str, top_k: int = 50) -> List[Tuple[int, float]]:
     q_terms = _tokenize(norm)
     if not q_terms:
         return []
+
     candidates = set()
     for t in q_terms:
         for idx in bm25_inverted.get(t, []):
             candidates.add(idx)
     if not candidates:
         candidates = set(range(len(bm25_docs)))
+
     scored = []
     for idx in candidates:
         s = _bm25_score_for_doc(q_terms, idx)
@@ -360,7 +355,7 @@ def bm25_search(query: str, top_k: int = 50) -> List[Tuple[int, float]]:
     scored.sort(key=lambda x: x[1], reverse=True)
     return scored[:top_k]
 
-# ---------------------------- Semantic-only ----------------------------
+# --------------------------- Semantic-only ---------------------------
 def search_knowledge_base(query: str, top_k: int = 10) -> dict:
     query_embedding = model.encode(query).tolist()
     res = collection.query(
@@ -391,13 +386,14 @@ def search_knowledge_base(query: str, top_k: int = 10) -> dict:
         "ids": ids,
     }
 
-# ---------------------------- Hybrid search (improved + exact-match rerank) ----------------------------
+# --------------------------- Hybrid search (improved + exact-match rerank) ---------------------------
 ACTION_SYNONYMS = {
-    "create": ["create", "creation", "add", "new", "generate"],
-    "update": ["update", "modify", "change", "edit"],
+    "create": ["create", "creation", "add", "new", "generate", "book", "schedule", "set up"],
+    "update": ["update", "modify", "change", "edit", "reschedule", "adjust", "move"],
     "delete": ["delete", "remove"],
     "navigate": ["navigate", "go to", "open"],
 }
+
 ERROR_INTENT_TERMS = [
     "error", "issue", "fail", "not working", "resolution", "fix",
     "permission", "permissions", "access", "no access", "authorization", "authorisation",
@@ -448,7 +444,8 @@ def _action_weight(text: str, actions: List[str]) -> float:
         for syn in ACTION_SYNONYMS.get(act, [act]):
             if syn in t:
                 score += 1.0
-    conflicts = {"create": ["delete"], "delete": ["create"], "update": ["delete"], "navigate": []}
+    # conflict matrix: penalize mismatched operations (e.g., user wants update but chunk talks about create)
+    conflicts = {"create": ["delete"], "delete": ["create"], "update": ["create", "delete"], "navigate": []}
     for act in actions:
         for bad in conflicts.get(act, []):
             for syn in ACTION_SYNONYMS.get(bad, [bad]):
@@ -482,7 +479,7 @@ def _intent_weight(meta: dict, user_intent: str) -> float:
         any(k in st for k in ["common errors", "known issues", "common issues", "errors", "escalation", "permissions", "access"])
         or ("permissions" in topic_list)
     ):
-        return 1.10  # stronger than before
+        return 1.10
     if user_intent == "steps" and any(k in st for k in ["inbound receiving", "receiving", "goods receipt", "grn"]):
         return 0.75
     return -0.2
@@ -568,13 +565,13 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
 
     union_ids = set(sem_ids) | set(bm25_id_to_norm.keys())
 
-    gamma = 0.30   # meta overlap
-    delta = 0.55   # intent boost (stronger)
-    epsilon = 0.30 # action weight
-    zeta = 0.65    # module weight
-    eta = 0.50     # phrase-level boost (stronger)
-    theta = 0.40   # heading alignment bonus
-    iota = 0.60    # literal query match boost (stronger)
+    gamma = 0.30  # meta overlap
+    delta = 0.55  # intent boost (stronger)
+    epsilon = 0.30  # action weight
+    zeta = 0.65  # module weight
+    eta = 0.50  # phrase-level boost (stronger)
+    theta = 0.40  # heading alignment bonus
+    iota = 0.60  # literal query match boost (stronger)
 
     combined_records_ext: List[Tuple[str, float, float, str, Dict[str, Any], float, float, float, float, float, float, float]] = []
     for cid in union_ids:
@@ -633,8 +630,7 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
         for rec in combined_records_ext:
             text_lower = (rec[3] or "").lower()
             if any(phrase in text_lower for phrase in [
-                norm_query,  # whole normalized query
-                # common 2-gram patterns extracted from the query
+                norm_query,
                 *(_make_ngrams([tok for tok in norm_query.split() if len(tok) > 2], 2))
             ]):
                 exact_hits.append(rec)
@@ -662,11 +658,13 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
         total_heading = sum(r[10] for r in recs)
         total_literal = sum(r[11] for r in recs)
         total_penalty = sum(min(0.0, r[6]) for r in recs) + sum(min(0.0, r[7]) for r in recs)
-        # Errors doc prior: if many chunks are from an errors/known issues section, add a bonus
+
+        # Errors doc prior: bonus for errors/known issues sections
         errors_section_bonus = 0.0
         if any("error" in ((r[4] or {}).get("section", "")).lower() or "known issues" in ((r[4] or {}).get("section", "")).lower()
                or "common issues" in ((r[4] or {}).get("section", "")).lower() for r in recs):
             errors_section_bonus = 0.5
+
         return (
             total_score
             + 0.4 * total_overlap
@@ -696,7 +694,6 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
 
     reordered = best_recs + other_recs
     top = reordered[:top_k]
-
     documents = [t[3] for t in top]
     metadatas = [t[4] for t in top]
     distances = [t[2] for t in top]
@@ -715,7 +712,7 @@ def hybrid_search_knowledge_base(query: str, top_k: int = 10, alpha: float = 0.6
         "actions": actions,
     }
 
-# ---------------------------- Section fetch helpers ----------------------------
+# --------------------------- Section fetch helpers ---------------------------
 def get_section_text(filename: str, section: str) -> str:
     texts: List[str] = []
     for d in bm25_docs:
@@ -759,7 +756,7 @@ def get_best_errors_section_text(filename: str) -> str:
                 texts.append(t)
     return "\n\n".join(texts).strip()
 
-# ---------------------------- Admin helpers ----------------------------
+# --------------------------- Admin helpers ---------------------------
 def get_kb_runtime_info() -> Dict[str, Any]:
     return {
         "chroma_path": CHROMA_PATH,