Update main.py

main.py

@@ -1,6 +1,4 @@
 
-# main_hugging_phase_recent.py
-
 import os
 import json
 import re
@@ -14,6 +12,8 @@ from pydantic import BaseModel
 from dotenv import load_dotenv
 from datetime import datetime
 
+# Import shared vocab from KB services
+from services.kb_creation import ACTION_SYNONYMS, MODULE_VOCAB
 from services.kb_creation import (
     collection,
     ingest_documents,
@@ -144,6 +144,20 @@ ERROR_FAMILY_SYNS = {
     ),
 }
 
+# ----- local extension so runtime filtering is precise even without re-ingest -----
+# (Does NOT override your KB synonyms—just augments them at runtime.)
+ACTION_SYNONYMS_EXT: Dict[str, List[str]] = {}
+for k, v in ACTION_SYNONYMS.items():
+    ACTION_SYNONYMS_EXT[k] = list(v)  # copy
+
+# Extend with SOP phrasing (appointments often say 'updation', 'deletion', 'reschedule')
+ACTION_SYNONYMS_EXT.setdefault("create", []).extend(["appointment creation", "create appointment"])
+ACTION_SYNONYMS_EXT.setdefault("update", []).extend([
+    "updation", "reschedule", "change time", "change date", "change slot",
+    "update time", "update date", "update slot", "update appointment", "edit appointment"
+])
+ACTION_SYNONYMS_EXT.setdefault("delete", []).extend(["deletion", "delete appointment", "cancel appointment"])
+
 
 def _detect_error_families(msg: str) -> list:
     """Return matching error family names found in the message (generic across SOPs)."""
@@ -157,6 +171,116 @@ def _detect_error_families(msg: str) -> list:
     return fams
 
 
+# --- Action-targeted steps selector (uses existing KB metadata) ---
+from services.kb_creation import bm25_docs, get_section_text
+
+def _get_steps_for_action(best_doc: str, actions: list) -> Optional[str]:
+    """
+    Return the full text of the steps section whose action_tag matches the user's intent.
+    e.g., actions=['update'] -> section: "Appointment schedule updation"
+          actions=['delete'] -> section: "Appointment deletion"
+          actions=['create'] -> section: "Appointment Creation"
+    """
+    if not best_doc or not actions:
+        return None
+    act_set = set(a.strip().lower() for a in actions if a)
+    # Collect candidate sections in this doc that are 'steps' and have an action_tag we need
+    candidates = []
+    for d in bm25_docs:
+        m = d.get("meta", {})
+        if m.get("filename") == best_doc and (m.get("intent_tag") == "steps"):
+            tag = (m.get("action_tag") or "").strip().lower()
+            if tag and tag in act_set:
+                candidates.append(m.get("section"))
+    # Prefer the first matched section with non-empty text
+    for title in candidates:
+        txt = get_section_text(best_doc, title)
+        if txt and txt.strip():
+            return txt.strip()
+    return None
+
+
+def _get_steps_for_action_global(actions: list, prefer_doc: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]:
+    """
+    Search ALL SOP docs for a 'steps' section whose action_tag matches one of `actions`.
+    Returns (doc_name, steps_text). Prefers sections from `prefer_doc` and the 'appointments' module.
+    """
+    if not actions:
+        return None, None
+    act_set = set(a.strip().lower() for a in actions if a)
+
+    candidates: List[Tuple[float, str, str]] = []  # (score, doc, text)
+    for d in bm25_docs:
+        m = d.get("meta", {}) or {}
+        if m.get("intent_tag") != "steps":
+            continue
+        tag = ((m.get("action_tag") or "").strip().lower())
+        if tag and tag in act_set:
+            doc = m.get("filename")
+            sec = (m.get("section") or "").strip()
+            txt = get_section_text(doc, sec)
+            if not txt or not txt.strip():
+                continue
+            score = 1.0
+            if prefer_doc and doc == prefer_doc:
+                score += 0.5
+            mtags = (m.get("module_tags") or "").lower()
+            if "appointments" in mtags:
+                score += 0.3
+            candidates.append((score, doc, txt.strip()))
+
+    if not candidates:
+        return None, None
+
+    candidates.sort(key=lambda x: x[0], reverse=True)
+    _, best_doc_global, best_text = candidates[0]
+    return best_doc_global, best_text
+
+# --- Default section picker when query doesn't reveal action ---
+def _pick_default_action_section(best_doc: str) -> Optional[str]:
+    """
+    If user actions are empty, prefer '...Creation' section,
+    else prefer '...Updation'/'...Update', else '...Deletion'/'...Cancel'.
+    Works generically for SOPs that use common headings.
+    """
+    order = ("creation", "updation", "update", "deletion", "delete", "cancel")
+    sections = []
+    for d in bm25_docs:
+        m = d.get("meta", {})
+        if m.get("filename") == best_doc and m.get("intent_tag") == "steps":
+            title = (m.get("section") or "").strip().lower()
+            if title:
+                sections.append(title)
+    for key in order:
+        for t in sections:
+            if key in t:
+                return t
+    return sections[0] if sections else None
+
+
+# --- Harvest 'Save' lines from ALL steps chunks in the doc (generic across SOPs) ---
+SAVE_SYNS = ("save", "saving", "save changes", "click save", "press save", "select save")
+
+def _find_save_lines_in_doc(best_doc: str, max_lines: int = 2) -> str:
+    """
+    Pulls up to max_lines lines that mention 'save' from any steps chunk in best_doc.
+    Returns a \n-joined string or empty if none found.
+    """
+    lines: List[str] = []
+    for d in bm25_docs:
+        m = d.get("meta", {})
+        if m.get("filename") != best_doc or m.get("intent_tag") != "steps":
+            continue
+        t = (d.get("text") or "").strip()
+        for ln in [x.strip() for x in t.splitlines() if x.strip()]:
+            low = ln.lower()
+            if any(s in low for s in SAVE_SYNS):
+                lines.append(ln)
+                if len(lines) >= max_lines:
+                    return "\n".join(lines)
+    return "\n".join(lines)
+
+
 def _is_domain_status_context(msg_norm: str) -> bool:
     if "status locked" in msg_norm or "locked status" in msg_norm:
         return True
@@ -678,7 +802,7 @@ def _set_incident_resolved(sys_id: str) -> bool:
         notes_field = os.getenv("SERVICENOW_RESOLUTION_NOTES_FIELD", "close_notes")
         payload_C = clean({
             "state": "6",
-            code_field: close_code_val,
+            code_field: close_notes_val,  # (if you have custom fields, adjust here)
             notes_field: close_notes_val,
             "caller_id": caller_sysid,
             "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
@@ -899,8 +1023,12 @@ async def chat_with_ai(input_data: ChatInput):
 
         selected = items[:max(1, 2)]
         context_raw = "\n\n---\n\n".join([s["text"] for s in selected]) if selected else ""
+
+        # Compute filter info for gating only; do NOT use the filtered text for steps
         filtered_text, filt_info = _filter_context_for_query(context_raw, input_data.user_message)
-        context = filtered_text
+        filtered_context = filtered_text
+
+        context = context_raw  # keep raw; we'll decide below
         context_found = bool(context.strip())
         best_distance = min([d for d in distances if d is not None], default=None) if distances else None
         best_combined = max([c for c in combined if c is not None], default=None) if combined else None
@@ -938,24 +1066,23 @@ async def chat_with_ai(input_data: ChatInput):
         if detected_intent == "neutral" and any(h in sec_title for h in PREREQ_HEADINGS):
             detected_intent = "prereqs"
 
-        # --- Steps nudge: "how to / perform" + receiving/inbound => steps intent
+        # --- Module-aware steps nudge (appointments, picking, shipping, etc.) ---
         STEPS_TERMS = ("how to", "procedure", "perform", "steps", "do", "navigate")
-        RECEIVING_TERMS = ("inbound", "receiving", "goods receipt", "grn")
-
+        ACTIONS_PRESENT = any(s in msg_low for syns in ACTION_SYNONYMS_EXT.values() for s in syns)
         mod_tags = ((top_meta or {}).get("module_tags") or "").lower()
 
-        looks_like_steps_query = any(t in msg_low for t in STEPS_TERMS)
-        looks_like_receiving = (
-            any(t in msg_low for t in RECEIVING_TERMS)
-            or "receiving" in mod_tags
-            or "inbound" in sec_title
-            or "receiving" in sec_title
+        MODULE_TOKENS = tuple({term for syns in MODULE_VOCAB.values() for term in syns})
+        looks_like_module = (
+            any(t in msg_low for t in MODULE_TOKENS)
+            or any(m in mod_tags for m in ["appointments", "receiving", "picking", "putaway", "shipping", "inventory", "replenishment"])
+            or any(m in sec_title for m in ["appointment", "appointments", "schedule", "dock", "door", "picking", "putaway", "shipping", "inventory"])
         )
+        looks_like_steps_query = any(t in msg_low for t in STEPS_TERMS) or ACTIONS_PRESENT
 
-        if detected_intent in ("neutral", "prereqs") and looks_like_steps_query and looks_like_receiving:
+        if detected_intent in ("neutral", "prereqs") and looks_like_steps_query and looks_like_module:
             detected_intent = "steps"
 
-        # --- Meaning-aware SOP gating ---
+        # --- Meaning-aware SOP gating (uses filter info) ---
         def _contains_any(s: str, keywords: tuple) -> bool:
             low = (s or "").lower()
             return any(k in low for k in keywords)
@@ -967,9 +1094,9 @@ async def chat_with_ai(input_data: ChatInput):
             "asn", "grn", "pick", "picking"
         )
         ACTION_OR_ERROR_TERMS = (
-            "how to", "procedure", "perform",  # added
+            "how to", "procedure", "perform",
             "close", "closing", "open", "navigate", "scan", "confirm", "generate", "update",
-            "receive", "receiving",            # added
+            "receive", "receiving",
             "error", "issue", "fail", "failed", "not working", "locked", "mismatch",
             "access", "permission", "status"
         )
@@ -986,8 +1113,7 @@ async def chat_with_ai(input_data: ChatInput):
         weak_domain_only = (mentions_domain and not has_any_action_or_error)
         low_context_hit = (matched_count < 2 and filter_mode in ("concise", "exact"))
 
-        # Bypass gate when strong steps signals are present for Receiving module
-        strong_steps_bypass = looks_like_steps_query and looks_like_receiving
+        strong_steps_bypass = looks_like_steps_query and looks_like_module
         strong_error_signal = len(_detect_error_families(msg_low)) > 0
         if (weak_domain_only or (low_context_hit and not combined_ok)) \
             and not strong_steps_bypass \
@@ -1016,71 +1142,220 @@ async def chat_with_ai(input_data: ChatInput):
                 },
             }
 
-        # Build SOP context if allowed
-        if is_perm_query:
-            detected_intent = "errors"
-
-        escalation_line = None  # SOP escalation candidate
-        full_errors = None      # keep for possible escalation extraction
+        # ---------- Build SOP context ----------
+        def _detect_action_from_query(q: str) -> Optional[str]:
+            qlow = (q or "").lower()
+            for act, syns in ACTION_SYNONYMS_EXT.items():
+                if any(s in qlow for s in syns):
+                    return act
+            return None
+
+        def _strip_boilerplate(raw_context: str) -> str:
+            """Remove document title/date/author/change-history noise from steps."""
+            MONTH_TERMS = ("january", "february", "march", "april", "may", "june",
+                           "july", "august", "september", "october", "november", "december")
+            lines = _normalize_lines(raw_context)
+            cleaned: List[str] = []
+            for ln in lines:
+                low = ln.lower()
+                is_change_hist = ("change history" in low) or ("initial draft" in low) or ("review" in low) or ("version" in low)
+                has_month_year = any(m in low for m in MONTH_TERMS) and bool(re.search(r"\b20\d{2}\b", low))
+                is_title_line = ("sop document" in low) or ("contents" in low)
+                if is_change_hist or has_month_year or is_title_line:
+                    continue
+                cleaned.append(ln)
+            return "\n".join(cleaned).strip()
+
+        def _extract_action_block(raw_context: str, target_act: Optional[str]) -> str:
+            """
+            Extract the contiguous block of lines for the target action (create/update/delete).
+            Start when a line mentions the target action OR looks procedural for 'create',
+            and stop ONLY when a line is a clear boundary:
+              - inline heading for another topic (e.g., 'Appointment schedule updation', 'Appointment deletion'), OR
+              - a line that strongly signals a different action (update/delete) via extended synonyms.
+            """
+            if not raw_context.strip():
+                return raw_context
+
+            lines = _normalize_lines(raw_context)
+            if not lines or not target_act:
+                return raw_context
+
+            INLINE_BOUNDARIES = (
+                "appointment schedule updation",
+                "schedule updation",
+                "appointment deletion",
+                "deletion",
+            )
+
+            other_terms: List[str] = []
+            for act, syns in ACTION_SYNONYMS_EXT.items():
+                if act != target_act:
+                    other_terms.extend(syns)
+            other_terms_low = set(t.lower() for t in other_terms)
+
+            def is_boundary_line(low: str) -> bool:
+                if any(h in low for h in INLINE_BOUNDARIES):
+                    return True
+                if any(t in low for t in other_terms_low):
+                    return True
+                return False
+
+            PROCEDURAL_VERBS = ("select", "choose", "click", "open", "add", "assign", "save",
+                                "navigate", "tag", "displayed", "triggered")
+            def is_procedural(low: str) -> bool:
+                return any(v in low for v in PROCEDURAL_VERBS)
+
+            target_terms_low = set(t.lower() for t in ACTION_SYNONYMS_EXT.get(target_act, []))
+
+            started = False
+            block: List[str] = []
+
+            for ln in lines:
+                low = ln.lower()
+
+                contains_target = any(t in low for t in target_terms_low)
+                if not started:
+                    if contains_target or (target_act == "create" and is_procedural(low)):
+                        started = True
+                        block.append(ln)
+                    continue
+
+                if is_boundary_line(low):
+                    break
+
+                block.append(ln)
+
+            return "\n".join(block).strip() if block else raw_context
+
+        def _filter_steps_by_action(raw_context: str, asked_act: Optional[str]) -> str:
+            cleaned = _strip_boilerplate(raw_context)
+            block = _extract_action_block(cleaned, asked_act)
+            if asked_act:
+                other_terms: List[str] = []
+                for act, syns in ACTION_SYNONYMS_EXT.items():
+                    if act != asked_act:
+                        other_terms.extend(syns)
+                lines = _normalize_lines(block)
+                lines = [ln for ln in lines if not any(t in ln.lower() for t in other_terms)]
+                block = "\n".join(lines).strip() if lines else block
+            return block
+
+        escalation_line = None
+        full_errors = None
         next_step_applied = False
         next_step_info: Dict[str, Any] = {}
 
-        if best_doc:
-            if detected_intent == "steps":
-                full_steps = get_best_steps_section_text(best_doc)
-                if not full_steps:
-                    sec = (top_meta or {}).get("section")
-                    if sec:
-                        full_steps = get_section_text(best_doc, sec)
+        if best_doc and detected_intent == "steps":
+            context_preformatted = False
+            full_steps = None
 
-                if full_steps:
-                    # Use numbered form only for matching; keep raw for full output
-                    numbered_full = _ensure_numbering(full_steps)
-                    next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
-
-                    if next_only is not None:
-                        # "what's next" mode
-                        if len(next_only) == 0:
-                            context = "You are at the final step of this SOP. No further steps."
-                            next_step_applied = True
-                            next_step_info = {"count": 0}
-                            context_preformatted = True
-                        else:
-                            context = _format_steps_as_numbered(next_only)
-                            next_step_applied = True
-                            next_step_info = {"count": len(next_only)}
-                            context_preformatted = True
+            # 1) Try by KB action tags
+            action_steps = _get_steps_for_action(best_doc, kb_results.get("actions", []))
+            if action_steps:
+                full_steps = action_steps
+            else:
+                if kb_results.get("actions"):
+                   alt_doc, alt_steps = _get_steps_for_action_global(kb_results["actions"], prefer_doc=best_doc)
+                   if alt_steps:
+                      best_doc = alt_doc   # switch to the doc that actually has the section
+                      full_steps = alt_steps
+                       
+                asked_action = _detect_action_from_query(input_data.user_message)
+                if not kb_results.get("actions") and asked_action:
+                   alt_doc, alt_steps = _get_steps_for_action_global([asked_action], prefer_doc=best_doc)
+                   if alt_steps:
+                      best_doc = alt_doc   # switch to the doc that actually has the section
+                      full_steps = alt_steps
+
+                if not full_steps:
+                   default_sec = _pick_default_action_section(best_doc)
+                   if default_sec:
+                      full_steps = get_section_text(best_doc, default_sec)
+                if not full_steps:
+                   full_steps = get_best_steps_section_text(best_doc)
+                if not full_steps:
+                   sec = (top_meta or {}).get("section")
+                   if sec:
+                      full_steps = get_section_text(best_doc, sec)
+    
+            if full_steps:
+                # Always add Save lines if present anywhere in the doc (independent of query wording)
+                save_lines = _find_save_lines_in_doc(best_doc, max_lines=2)
+                if save_lines:
+                    low_steps = (full_steps or "").lower()
+                    if not any(s in low_steps for s in SAVE_SYNS):
+                        full_steps = (full_steps or "").rstrip() + "\n" + save_lines
+
+                asked_action = _detect_action_from_query(input_data.user_message)
+                full_steps = _filter_steps_by_action(full_steps, asked_action)
+
+                numbered_full = _ensure_numbering(full_steps)
+                next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
+
+                if next_only is not None:
+                    if len(next_only) == 0:
+                        context = "You are at the final step of this SOP. No further steps."
+                        next_step_applied = True
+                        next_step_info = {"count": 0}
+                        context_preformatted = True
                     else:
-                        # Normal mode: return the full SOP section (raw),
-                        # and we'll number it below once.
-                        context = full_steps
-                        context_preformatted = False
-
-            elif detected_intent == "errors":
-                full_errors = get_best_errors_section_text(best_doc)
-                if full_errors:
-                    ctx_err = _extract_errors_only(full_errors, max_lines=30)
-                    if is_perm_query:
-                        context = _filter_permission_lines(ctx_err, max_lines=6)
+                        context = _format_steps_as_numbered(next_only)
+                        next_step_applied = True
+                        next_step_info = {"count": len(next_only)}
+                        context_preformatted = True
+                else:
+                    context = full_steps
+                    context_preformatted = False
+
+                # Clear filter info for debug clarity
+                filt_info = {'mode': None, 'matched_count': None, 'all_sentences': None}
+                context_found = True
+
+        elif best_doc and detected_intent == "errors":
+            full_errors = get_best_errors_section_text(best_doc)
+            if full_errors:
+                ctx_err = _extract_errors_only(full_errors, max_lines=30)
+                if is_perm_query:
+                    context = _filter_permission_lines(ctx_err, max_lines=6)
+                else:
+                    is_specific_error = len(_detect_error_families(msg_low)) > 0
+                    if is_specific_error:
+                        context = _filter_error_lines_by_query(ctx_err, input_data.user_message, max_lines=1)
                     else:
-                        # Decide specific vs generic:
-                        is_specific_error = len(_detect_error_families(msg_low)) > 0
-                        if is_specific_error:
-                            context = _filter_error_lines_by_query(ctx_err, input_data.user_message, max_lines=1)
-                        else:
-                            all_lines: List[str] = _normalize_lines(ctx_err)
-                            error_bullets = [ln for ln in all_lines if re.match(r"^\s*[-*\u2022]\s*", ln) or (":" in ln)]
-                            context = "\n".join(error_bullets[:8]).strip()
-                            assist_followup = (
-                                "Please tell me which error above matches your screen (paste the exact text), "
-                                "or share a screenshot. I can guide you further or raise a ServiceNow ticket."
-                            )
-                    escalation_line = _extract_escalation_line(full_errors)
-
-            elif detected_intent == "prereqs":
-                full_prereqs = _find_prereq_section_text(best_doc)
-                if full_prereqs:
-                    context = full_prereqs.strip()
+                        all_lines: List[str] = _normalize_lines(ctx_err)
+                        error_bullets = [ln for ln in all_lines if re.match(r"^\s*[-*\u2022]\s*", ln) or (":" in ln)]
+                        context = "\n".join(error_bullets[:8]).strip()
+                        assist_followup = (
+                            "Please tell me which error above matches your screen (paste the exact text), "
+                            "or share a screenshot. I can guide you further or raise a ServiceNow ticket."
+                        )
+                escalation_line = _extract_escalation_line(full_errors)
+            else:
+                full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
+                if full_steps:
+                    asked_action = _detect_action_from_query(input_data.user_message)
+                    full_steps = _filter_steps_by_action(full_steps, asked_action)
+                    context = full_steps
+                    detected_intent = "steps"
+                    context_preformatted = False
+
+        elif best_doc and detected_intent == "prereqs":
+            full_prereqs = _find_prereq_section_text(best_doc)
+            if full_prereqs:
+                context = full_prereqs.strip()
+            else:
+                full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
+                if full_steps:
+                    asked_action = _detect_action_from_query(input_data.user_message)
+                    full_steps = _filter_steps_by_action(full_steps, asked_action)
+                    context = full_steps
+                    detected_intent = "steps"
+                    context_preformatted = False
+
+        else:
+            # Neutral or other intents: use filtered context
+            context = filtered_context
 
         language_hint = _detect_language_hint(input_data.user_message)
         lang_line = f"Respond in {language_hint}." if language_hint else "Respond in a clear, polite tone."
@@ -1116,8 +1391,6 @@ Return ONLY the rewritten guidance."""
 
         # Deterministic local formatting
         if detected_intent == "steps":
-            # If we trimmed to next steps, 'context' is already formatted (or a sentence).
-            # Only number when returning full SOP raw text.
             if ('context_preformatted' in locals()) and context_preformatted:
                 bot_text = context
             else:
@@ -1152,9 +1425,14 @@ Return ONLY the rewritten guidance."""
                     "Share a bit more detail (module/screen/error), or say ‘create ticket’."
                 )
 
-        short_query = len((input_data.user_message or "").split()) <= 4
-        gate_combined_ok = 0.60 if short_query else 0.55
-        status = "OK" if (best_combined is not None and best_combined >= gate_combined_ok) else "PARTIAL"
+        # Status: mark OK when we served steps successfully
+        if detected_intent == "steps" and bot_text.strip():
+            status = "OK"
+        else:
+            short_query = len((input_data.user_message or "").split()) <= 4
+            gate_combined_ok = 0.60 if short_query else 0.55
+            status = "OK" if (best_combined is not None and best_combined >= gate_combined_ok) else "PARTIAL"
+
         lower = (bot_text or "").lower()
         if ("partial" in lower) or ("may be partial" in lower) or ("closest" in lower) or ("may not fully" in lower):
             status = "PARTIAL"
@@ -1175,8 +1453,8 @@ Return ONLY the rewritten guidance."""
                 "best_distance": best_distance,
                 "best_combined": best_combined,
                 "http_status": http_code,
-                "filter_mode": filt_info.get("mode"),
-                "matched_count": filt_info.get("matched_count"),
+                "filter_mode": (filt_info.get("mode") if filt_info else None),
+                "matched_count": (filt_info.get("matched_count") if filt_info else None),
                 "user_intent": detected_intent,
                 "best_doc": best_doc,
                 "next_step": {