Update main.py

main.py

@@ -227,14 +227,13 @@ def _get_steps_for_action_global(actions: list, prefer_doc: Optional[str] = None
     candidates.sort(key=lambda x: x[0], reverse=True)
     _, best_doc_global, best_text = candidates[0]
     return best_doc_global, best_text
-    
+
 def _pick_default_action_section(best_doc: str) -> Optional[str]:
     """
     Backward-compatible wrapper: prefer-action version with no preference.
     """
     return _pick_default_action_section_with_preference(best_doc, None)
 
-
 def _pick_default_action_section_with_preference(best_doc: str, prefer_action: Optional[str]) -> Optional[str]:
     """
     Generic: if there are sections tagged with the requested action (via action_tag),
@@ -265,8 +264,7 @@ def _pick_default_action_section_with_preference(best_doc: str, prefer_action: O
                 return t
     return sections[0] if sections else None
 
-# ------------------------------ Action -> section selector (exact match by section title) ------------------------------
-# Maps user action to likely section name tokens (lowercase)
+# ------------------------------ Action -> section selector (optional fallback by title) ------------------------------
 ACTION_SECTION_KEYS = {
     "create": ("create", "creation", "appointment creation", "new appointment", "book", "schedule"),
     "update": ("update", "updation", "reschedule", "change", "modify", "edit"),
@@ -274,17 +272,11 @@ ACTION_SECTION_KEYS = {
 }
 
 def _pick_action_section(best_doc: str, action: Optional[str]) -> Optional[str]:
-    """
-    Return the exact section title (original casing) inside `best_doc` whose name
-    contains any token for the given action. Falls back to None.
-    """
     if not best_doc or not action:
         return None
     keys = ACTION_SECTION_KEYS.get(action.lower(), ())
     if not keys:
         return None
-
-    # scan available sections for this doc
     candidates = []
     for d in bm25_docs:
         m = d.get("meta", {}) or {}
@@ -294,32 +286,24 @@ def _pick_action_section(best_doc: str, action: Optional[str]) -> Optional[str]:
                 continue
             low = sec.lower()
             if any(k in low for k in keys):
-                # prefer exact action match over generic words
                 score = 1.0
                 if action.lower() in low:
                     score += 0.5
-                # appointments module slight boost
                 mtags = (m.get("module_tags") or "").lower()
                 if "appointments" in mtags:
                     score += 0.2
                 candidates.append((score, sec))
-
     if not candidates:
         return None
-
-    # pick best scored section title
     candidates.sort(key=lambda x: x[0], reverse=True)
-    return candidates[0][1]  # exact section title
+    return candidates[0][1]
 
 def _get_steps_text_for_action(best_doc: str, action: Optional[str]) -> Optional[str]:
-    """
-    Use the section title returned by `_pick_action_section` to fetch its text.
-    """
     sec = _pick_action_section(best_doc, action)
     if not sec:
         return None
     txt = get_section_text(best_doc, sec)
-    return (txt or "").strip() or None    
+    return (txt or "").strip() or None
 
 # ------------------------------ Save lines helpers ------------------------------
 SAVE_SYNS = ("save", "saving", "save changes", "click save", "press save", "select save")
@@ -334,30 +318,52 @@ def _find_save_lines_in_section(section_text: str, max_lines: int = 2) -> str:
                 break
     return "\n".join(lines)
 
-# ------------------------------ Boundary cutter to avoid bleed ------------------------------
-BOUNDARY_HEADINGS = (
-    "appointment schedule updation","schedule updation","updation","update",
-    "appointment deletion","deletion","delete","cancel",
-    "escalation","common errors","known issues","errors & resolution",
-    "pre-requisites","prerequisites"
-)
+# ------------------------------ Generic boundary cutter (metadata-driven) ------------------------------
+def _build_doc_section_index(best_doc: str) -> Dict[str, Optional[str]]:
+    """
+    Build a dictionary for the given doc:
+    { lower(section_title): lower(action_tag or None) }
+    """
+    index: Dict[str, Optional[str]] = {}
+    for d in bm25_docs:
+        m = d.get("meta", {}) or {}
+        if m.get("filename") == best_doc and m.get("intent_tag") == "steps":
+            sec = (m.get("section") or "").strip()
+            tag = (m.get("action_tag") or "").strip().lower() or None
+            if sec:
+                index[sec.lower()] = tag
+    return index
 
-def _cut_at_next_boundary(section_text: str, current_action: Optional[str]) -> str:
+def _cut_at_next_boundary_generic(section_text: str, best_doc: str, current_action: Optional[str]) -> str:
+    """
+    Stop when we hit any known section heading in the same doc.
+    If current_action is set, stop only when the next heading belongs to a different action_tag.
+    Fully generic: no hard-coded SOP words.
+    """
     if not (section_text or "").strip():
         return section_text
+
+    index = _build_doc_section_index(best_doc)
+    known_headings = set(index.keys())
     lines = [ln for ln in (section_text or "").splitlines() if ln.strip()]
     out: List[str] = []
+
     for ln in lines:
         low = ln.lower().strip()
-        is_heading_like = (":" in ln and len(ln.split(":")[0]) <= 80) or any(k in low for k in BOUNDARY_HEADINGS)
-        if is_heading_like:
-            if current_action == "create" and any(k in low for k in ("update","updation","deletion","delete","cancel")):
-                break
-            if current_action == "update" and any(k in low for k in ("deletion","delete","cancel")):
+
+        matched_heading = None
+        for h in known_headings:
+            if h in low:
+                matched_heading = h
                 break
-            if current_action is None and any(k in low for k in BOUNDARY_HEADINGS):
+
+        if matched_heading:
+            next_action = index.get(matched_heading)
+            if (current_action and next_action and next_action != current_action) or (current_action is None):
                 break
+
         out.append(ln)
+
     return "\n".join(out).strip()
 
 # ------------------------------ Text normalization / numbering ------------------------------
@@ -602,58 +608,6 @@ def _build_tracking_descriptions(issue_text: str, resolved_text: str) -> Tuple[s
         f"Tracking record created automatically by NOVA."
     ).strip()
     return short_desc, long_desc
-    
-def _build_doc_section_index(best_doc: str) -> Dict[str, Optional[str]]:
-    """
-    Build a dictionary for the given doc:
-    { lower(section_title): lower(action_tag or None) }
-    """
-    index: Dict[str, Optional[str]] = {}
-    for d in bm25_docs:
-        m = d.get("meta", {}) or {}
-        if m.get("filename") == best_doc and m.get("intent_tag") == "steps":
-            sec = (m.get("section") or "").strip()
-            tag = (m.get("action_tag") or "").strip().lower() or None
-            if sec:
-                index[sec.lower()] = tag
-    return index
-
-def _cut_at_next_boundary_generic(section_text: str, best_doc: str, current_action: Optional[str]) -> str:
-    """
-    Stop when we hit any known section heading in the same doc.
-    If current_action is set, stop only when the next heading belongs to a different action_tag.
-    Fully generic: no hard-coded SOP words.
-    """
-    if not (section_text or "").strip():
-        return section_text
-
-    index = _build_doc_section_index(best_doc)
-    known_headings = set(index.keys())
-    lines = [ln for ln in (section_text or "").splitlines() if ln.strip()]
-    out: List[str] = []
-
-    for ln in lines:
-        low = ln.lower().strip()
-
-        # If any known heading appears in the line, consider it a boundary.
-        # We use substring match so headings like "Update: ..." or "Delete – ..." work.
-        matched_heading = None
-        for h in known_headings:
-            if h in low:
-                matched_heading = h
-                break
-
-        if matched_heading:
-            next_action = index.get(matched_heading)
-            # If we know the current action, break only when the upcoming heading is a different action section.
-            # If current_action is None, break at the first heading we encounter.
-            if (current_action and next_action and next_action != current_action) or (current_action is None):
-                break
-
-        out.append(ln)
-
-    return "\n".join(out).strip()
-
 
 # ------------------------------ Incident helpers ------------------------------
 def _is_incident_intent(msg_norm: str) -> bool:
@@ -701,108 +655,6 @@ def _has_negation_resolved(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in neg_phrases)
 
-def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str, Any]]:
-    STRICT_OVERLAP = 3
-    MAX_SENTENCES_STRICT = 4
-    MAX_SENTENCES_CONCISE = 3
-
-    def _norm(text: str) -> str:
-        t = (text or "").lower()
-        t = re.sub(r"[^\w\s]", " ", t)
-        t = re.sub(r"\s+", " ", t).strip()
-        return t
-
-    def _split_sentences(ctx: str) -> List[str]:
-        raw_sents = re.split(r"(?<=[.!?])\s+|\n+|-\s*|\*\s*", ctx or "")
-        return [s.strip() for s in raw_sents if s and len(s.strip()) > 2]
-
-    ctx = (context or "").strip()
-    if not ctx or not query:
-        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
-    q_norm = _norm(query)
-    q_terms = [t for t in q_norm.split() if len(t) > 2]
-    if not q_terms:
-        return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
-    sentences = _split_sentences(ctx)
-    matched_exact, matched_any = [], []
-    for s in sentences:
-        s_norm = _norm(s)
-        is_bullet = bool(re.match(r"^[\-\*]\s*", s))
-        overlap = sum(1 for t in q_terms if t in s_norm) + (1 if is_bullet else 0)
-        if overlap >= STRICT_OVERLAP:
-            matched_exact.append(s)
-        elif overlap > 0:
-            matched_any.append(s)
-    if matched_exact:
-        kept = matched_exact[:MAX_SENTENCES_STRICT]
-        return "\n".join(kept).strip(), {'mode': 'exact', 'matched_count': len(kept), 'all_sentences': len(sentences)}
-    if matched_any:
-        kept = matched_any[:MAX_SENTENCES_CONCISE]
-        return "\n".join(kept).strip(), {'mode': 'concise', 'matched_count': len(kept), 'all_sentences': len(sentences)}
-    kept = sentences[:MAX_SENTENCES_CONCISE]
-    return "\n".join(kept).strip(), {'mode': 'concise', 'matched_count': 0, 'all_sentences': len(sentences)}
-
-def _extract_errors_only(text: str, max_lines: int = 12) -> str:
-    kept: List[str] = []
-    for ln in _normalize_lines(text):
-        if re.match(r"^\s*[-*\u2022]\s*", ln) or (":" in ln):
-            kept.append(ln)
-            if len(kept) >= max_lines:
-                break
-    return "\n".join(kept).strip() if kept else (text or "").strip()
-
-def _filter_permission_lines(text: str, max_lines: int = 6) -> str:
-    PERM_SYNONYMS = (
-        "permission", "permissions", "access", "authorization", "authorisation",
-        "role", "role mapping", "security profile", "not allowed", "not authorized", "denied", "insufficient"
-    )
-    kept: List[str] = []
-    for ln in _normalize_lines(text):
-        low = ln.lower()
-        if any(k in low for k in PERM_SYNONYMS):
-            kept.append(ln)
-        if len(kept) >= max_lines:
-            break
-    return "\n".join(kept).strip() if kept else (text or "").strip()
-
-def _extract_escalation_line(text: str) -> Optional[str]:
-    if not text:
-        return None
-    lines = _normalize_lines(text)
-    if not lines:
-        return None
-    start_idx = None
-    for i, ln in enumerate(lines):
-        low = ln.lower()
-        if "escalation" in low or "escalation path" in low or "escalate" in low:
-            start_idx = i
-            break
-    block = []
-    if start_idx is not None:
-        for j in range(start_idx, min(len(lines), start_idx + 6)):
-            if not lines[j].strip():
-                break
-            block.append(lines[j].strip())
-    else:
-        block = [ln.strip() for ln in lines if ("->" in ln or "→" in ln)]
-    if not block:
-        return None
-    text_block = " ".join(block)
-    m = re.search(r"escalation[^:]*:\s*(.+)", text_block, flags=re.IGNORECASE)
-    path = m.group(1).strip() if m else None
-    if not path:
-        arrow_lines = [ln for ln in block if ("->" in ln or "→" in ln)]
-        if arrow_lines:
-            path = arrow_lines[0]
-    if not path:
-        m2 = re.search(r"(operator.*?administrator|operator.*)", text_block, flags=re.IGNORECASE)
-        path = m2.group(1).strip() if m2 else None
-    if not path:
-        return None
-    path = path.replace("->", "→").strip()
-    path = re.sub(r"^(?i:escalation\s*path)\s*:\s*", "", path).strip()
-    return f"If you want to escalate the issue, follow: {path}"
-
 # ------------------------------ Health ------------------------------
 @app.get("/")
 async def health_check():
@@ -853,12 +705,6 @@ async def chat_with_ai(input_data: ChatInput):
             except Exception:
                 is_llm_resolved = False
 
-        def _has_negation_resolved(msg_norm: str) -> bool:
-            return any(p in msg_norm for p in [
-                "not resolved", "issue not resolved", "still not working", "not working",
-                "didn't work", "doesn't work", "no change", "not fixed", "still failing", "failed again", "broken", "fail",
-            ])
-
         if _has_negation_resolved(msg_norm):
             is_llm_resolved = False
         if (not _has_negation_resolved(msg_norm)) and (any(p in msg_norm for p in [
@@ -1162,42 +1008,34 @@ async def chat_with_ai(input_data: ChatInput):
             context_preformatted = False
             full_steps = None
 
-# Detect asked action generically using ACTION_SYNONYMS_EXT (already defined)
-            asked_action = _detect_action_from_query(input_data.user_message)  # returns 'create'/'update'/'delete'/None
+            # Detect asked action generically using ACTION_SYNONYMS_EXT (already defined)
+            asked_action = _detect_action_from_query(input_data.user_message)  # 'create'/'update'/'delete'/None
 
+            # 1) Try KB-tagged action section (existing)
             action_steps = _get_steps_for_action(best_doc, kb_results.get("actions", []))
             if action_steps:
                 full_steps = action_steps
             else:
+                # 2) Global action lookup (existing)
                 if kb_results.get("actions"):
                     alt_doc, alt_steps = _get_steps_for_action_global(kb_results["actions"], prefer_doc=best_doc)
                     if alt_steps:
                         best_doc = alt_doc
                         full_steps = alt_steps
-               
-                if not full_steps and asked_action:
-                   via_tag_steps = _get_steps_by_action_tag(best_doc, asked_action)
-                   if via_tag_steps:
-                      full_steps = via_tag_steps
-                if not full_steps:
-                   default_sec = _pick_default_action_section_with_preference(best_doc, asked_action)
-                   if default_sec:
-                      full_steps = get_section_text(best_doc, default_sec)
 
+                # 3) Generic: exact section by meta.action_tag in SAME best_doc
                 if not full_steps and asked_action:
-                    exact_action_steps = _get_steps_text_for_action(best_doc, asked_action)
-                    if exact_action_steps:
-                        full_steps = exact_action_steps
+                    via_tag_steps = _get_steps_by_action_tag(best_doc, asked_action)
+                    if via_tag_steps:
+                        full_steps = via_tag_steps
 
-                if not full_steps and asked_action:
-                    alt_doc2, alt_steps2 = _get_steps_for_action_global([asked_action], prefer_doc=best_doc)
-                    if alt_steps2:
-                        best_doc = alt_doc2
-                        full_steps = alt_steps2
+                # 4) Prefer requested action when defaulting
                 if not full_steps:
-                    default_sec = _pick_default_action_section(best_doc)
+                    default_sec = _pick_default_action_section_with_preference(best_doc, asked_action)
                     if default_sec:
                         full_steps = get_section_text(best_doc, default_sec)
+
+                # 5) Existing fallbacks
                 if not full_steps:
                     full_steps = get_best_steps_section_text(best_doc)
                 if not full_steps:
@@ -1206,10 +1044,10 @@ async def chat_with_ai(input_data: ChatInput):
                         full_steps = get_section_text(best_doc, sec)
 
             if full_steps:
-                # NEW: cut at boundary so 'Creation' doesn't bleed into 'Updation/Delete'
-                asked_action = _detect_action_from_query(input_data.user_message)
-                full_steps = _cut_at_next_boundary_generic(full_steps, best_doc, asked_action)             
-                # NEW: scope 'Save' to this section
+                # Generic boundary cutter (prevents create/update/delete bleed)
+                full_steps = _cut_at_next_boundary_generic(full_steps, best_doc, asked_action)
+
+                # Scope 'Save' lines to current section
                 save_local = _find_save_lines_in_section(full_steps, max_lines=2)
                 if save_local:
                     low_steps = (full_steps or "").lower()
@@ -1218,7 +1056,7 @@ async def chat_with_ai(input_data: ChatInput):
 
                 # Number + Next steps
                 numbered_full = _ensure_numbering(full_steps)
-                next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
+                next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.30)
                 if next_only is not None:
                     if len(next_only) == 0:
                         context = "You are at the final step of this SOP. No further steps."
@@ -1233,9 +1071,10 @@ async def chat_with_ai(input_data: ChatInput):
                 else:
                     context = full_steps
                     context_preformatted = False
+
                 context_found = True
                 filt_info = {'mode': None, 'matched_count': None, 'all_sentences': None}
-                
+
         elif best_doc and detected_intent == "errors":
             full_errors = get_best_errors_section_text(best_doc)
             if full_errors: