Update main.py

main.py

@@ -1,4 +1,6 @@
 
+# main_hugging_phase_recent.py
+
 import os
 import json
 import re
@@ -12,8 +14,6 @@ from pydantic import BaseModel
 from dotenv import load_dotenv
 from datetime import datetime
 
-# Import shared vocab from KB services
-from services.kb_creation import ACTION_SYNONYMS, MODULE_VOCAB
 from services.kb_creation import (
     collection,
     ingest_documents,
@@ -144,20 +144,6 @@ ERROR_FAMILY_SYNS = {
     ),
 }
 
-# ----- local extension so runtime filtering is precise even without re-ingest -----
-# (Does NOT override your KB synonyms—just augments them at runtime.)
-ACTION_SYNONYMS_EXT: Dict[str, List[str]] = {}
-for k, v in ACTION_SYNONYMS.items():
-    ACTION_SYNONYMS_EXT[k] = list(v)  # copy
-
-# Extend with SOP phrasing (appointments often say 'updation', 'deletion', 'reschedule')
-ACTION_SYNONYMS_EXT.setdefault("create", []).extend(["appointment creation", "create appointment"])
-ACTION_SYNONYMS_EXT.setdefault("update", []).extend([
-    "updation", "reschedule", "change time", "change date", "change slot",
-    "update time", "update date", "update slot", "update appointment", "edit appointment"
-])
-ACTION_SYNONYMS_EXT.setdefault("delete", []).extend(["deletion", "delete appointment", "cancel appointment"])
-
 
 def _detect_error_families(msg: str) -> list:
     """Return matching error family names found in the message (generic across SOPs)."""
@@ -171,116 +157,6 @@ def _detect_error_families(msg: str) -> list:
     return fams
 
 
-# --- Action-targeted steps selector (uses existing KB metadata) ---
-from services.kb_creation import bm25_docs, get_section_text
-
-def _get_steps_for_action(best_doc: str, actions: list) -> Optional[str]:
-    """
-    Return the full text of the steps section whose action_tag matches the user's intent.
-    e.g., actions=['update'] -> section: "Appointment schedule updation"
-          actions=['delete'] -> section: "Appointment deletion"
-          actions=['create'] -> section: "Appointment Creation"
-    """
-    if not best_doc or not actions:
-        return None
-    act_set = set(a.strip().lower() for a in actions if a)
-    # Collect candidate sections in this doc that are 'steps' and have an action_tag we need
-    candidates = []
-    for d in bm25_docs:
-        m = d.get("meta", {})
-        if m.get("filename") == best_doc and (m.get("intent_tag") == "steps"):
-            tag = (m.get("action_tag") or "").strip().lower()
-            if tag and tag in act_set:
-                candidates.append(m.get("section"))
-    # Prefer the first matched section with non-empty text
-    for title in candidates:
-        txt = get_section_text(best_doc, title)
-        if txt and txt.strip():
-            return txt.strip()
-    return None
-
-
-def _get_steps_for_action_global(actions: list, prefer_doc: Optional[str] = None) -> Tuple[Optional[str], Optional[str]]:
-    """
-    Search ALL SOP docs for a 'steps' section whose action_tag matches one of `actions`.
-    Returns (doc_name, steps_text). Prefers sections from `prefer_doc` and the 'appointments' module.
-    """
-    if not actions:
-        return None, None
-    act_set = set(a.strip().lower() for a in actions if a)
-
-    candidates: List[Tuple[float, str, str]] = []  # (score, doc, text)
-    for d in bm25_docs:
-        m = d.get("meta", {}) or {}
-        if m.get("intent_tag") != "steps":
-            continue
-        tag = ((m.get("action_tag") or "").strip().lower())
-        if tag and tag in act_set:
-            doc = m.get("filename")
-            sec = (m.get("section") or "").strip()
-            txt = get_section_text(doc, sec)
-            if not txt or not txt.strip():
-                continue
-            score = 1.0
-            if prefer_doc and doc == prefer_doc:
-                score += 0.5
-            mtags = (m.get("module_tags") or "").lower()
-            if "appointments" in mtags:
-                score += 0.3
-            candidates.append((score, doc, txt.strip()))
-
-    if not candidates:
-        return None, None
-
-    candidates.sort(key=lambda x: x[0], reverse=True)
-    _, best_doc_global, best_text = candidates[0]
-    return best_doc_global, best_text
-
-# --- Default section picker when query doesn't reveal action ---
-def _pick_default_action_section(best_doc: str) -> Optional[str]:
-    """
-    If user actions are empty, prefer '...Creation' section,
-    else prefer '...Updation'/'...Update', else '...Deletion'/'...Cancel'.
-    Works generically for SOPs that use common headings.
-    """
-    order = ("creation", "updation", "update", "deletion", "delete", "cancel")
-    sections = []
-    for d in bm25_docs:
-        m = d.get("meta", {})
-        if m.get("filename") == best_doc and m.get("intent_tag") == "steps":
-            title = (m.get("section") or "").strip().lower()
-            if title:
-                sections.append(title)
-    for key in order:
-        for t in sections:
-            if key in t:
-                return t
-    return sections[0] if sections else None
-
-
-# --- Harvest 'Save' lines from ALL steps chunks in the doc (generic across SOPs) ---
-SAVE_SYNS = ("save", "saving", "save changes", "click save", "press save", "select save")
-
-def _find_save_lines_in_doc(best_doc: str, max_lines: int = 2) -> str:
-    """
-    Pulls up to max_lines lines that mention 'save' from any steps chunk in best_doc.
-    Returns a \n-joined string or empty if none found.
-    """
-    lines: List[str] = []
-    for d in bm25_docs:
-        m = d.get("meta", {})
-        if m.get("filename") != best_doc or m.get("intent_tag") != "steps":
-            continue
-        t = (d.get("text") or "").strip()
-        for ln in [x.strip() for x in t.splitlines() if x.strip()]:
-            low = ln.lower()
-            if any(s in low for s in SAVE_SYNS):
-                lines.append(ln)
-                if len(lines) >= max_lines:
-                    return "\n".join(lines)
-    return "\n".join(lines)
-
-
 def _is_domain_status_context(msg_norm: str) -> bool:
     if "status locked" in msg_norm or "locked status" in msg_norm:
         return True
@@ -802,7 +678,7 @@ def _set_incident_resolved(sys_id: str) -> bool:
         notes_field = os.getenv("SERVICENOW_RESOLUTION_NOTES_FIELD", "close_notes")
         payload_C = clean({
             "state": "6",
-            code_field: close_notes_val,  # (if you have custom fields, adjust here)
+            code_field: close_code_val,
             notes_field: close_notes_val,
             "caller_id": caller_sysid,
             "resolved_at": datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
@@ -1023,12 +899,8 @@ async def chat_with_ai(input_data: ChatInput):
 
         selected = items[:max(1, 2)]
         context_raw = "\n\n---\n\n".join([s["text"] for s in selected]) if selected else ""
-
-        # Compute filter info for gating only; do NOT use the filtered text for steps
         filtered_text, filt_info = _filter_context_for_query(context_raw, input_data.user_message)
-        filtered_context = filtered_text
-
-        context = context_raw  # keep raw; we'll decide below
+        context = filtered_text
         context_found = bool(context.strip())
         best_distance = min([d for d in distances if d is not None], default=None) if distances else None
         best_combined = max([c for c in combined if c is not None], default=None) if combined else None
@@ -1066,23 +938,24 @@ async def chat_with_ai(input_data: ChatInput):
         if detected_intent == "neutral" and any(h in sec_title for h in PREREQ_HEADINGS):
             detected_intent = "prereqs"
 
-        # --- Module-aware steps nudge (appointments, picking, shipping, etc.) ---
+        # --- Steps nudge: "how to / perform" + receiving/inbound => steps intent
         STEPS_TERMS = ("how to", "procedure", "perform", "steps", "do", "navigate")
-        ACTIONS_PRESENT = any(s in msg_low for syns in ACTION_SYNONYMS_EXT.values() for s in syns)
+        RECEIVING_TERMS = ("inbound", "receiving", "goods receipt", "grn")
+
         mod_tags = ((top_meta or {}).get("module_tags") or "").lower()
 
-        MODULE_TOKENS = tuple({term for syns in MODULE_VOCAB.values() for term in syns})
-        looks_like_module = (
-            any(t in msg_low for t in MODULE_TOKENS)
-            or any(m in mod_tags for m in ["appointments", "receiving", "picking", "putaway", "shipping", "inventory", "replenishment"])
-            or any(m in sec_title for m in ["appointment", "appointments", "schedule", "dock", "door", "picking", "putaway", "shipping", "inventory"])
+        looks_like_steps_query = any(t in msg_low for t in STEPS_TERMS)
+        looks_like_receiving = (
+            any(t in msg_low for t in RECEIVING_TERMS)
+            or "receiving" in mod_tags
+            or "inbound" in sec_title
+            or "receiving" in sec_title
         )
-        looks_like_steps_query = any(t in msg_low for t in STEPS_TERMS) or ACTIONS_PRESENT
 
-        if detected_intent in ("neutral", "prereqs") and looks_like_steps_query and looks_like_module:
+        if detected_intent in ("neutral", "prereqs") and looks_like_steps_query and looks_like_receiving:
             detected_intent = "steps"
 
-        # --- Meaning-aware SOP gating (uses filter info) ---
+        # --- Meaning-aware SOP gating ---
         def _contains_any(s: str, keywords: tuple) -> bool:
             low = (s or "").lower()
             return any(k in low for k in keywords)
@@ -1094,9 +967,9 @@ async def chat_with_ai(input_data: ChatInput):
             "asn", "grn", "pick", "picking"
         )
         ACTION_OR_ERROR_TERMS = (
-            "how to", "procedure", "perform",
+            "how to", "procedure", "perform",  # added
             "close", "closing", "open", "navigate", "scan", "confirm", "generate", "update",
-            "receive", "receiving",
+            "receive", "receiving",            # added
             "error", "issue", "fail", "failed", "not working", "locked", "mismatch",
             "access", "permission", "status"
         )
@@ -1113,7 +986,8 @@ async def chat_with_ai(input_data: ChatInput):
         weak_domain_only = (mentions_domain and not has_any_action_or_error)
         low_context_hit = (matched_count < 2 and filter_mode in ("concise", "exact"))
 
-        strong_steps_bypass = looks_like_steps_query and looks_like_module
+        # Bypass gate when strong steps signals are present for Receiving module
+        strong_steps_bypass = looks_like_steps_query and looks_like_receiving
         strong_error_signal = len(_detect_error_families(msg_low)) > 0
         if (weak_domain_only or (low_context_hit and not combined_ok)) \
             and not strong_steps_bypass \
@@ -1142,220 +1016,71 @@ async def chat_with_ai(input_data: ChatInput):
                 },
             }
 
-        # ---------- Build SOP context ----------
-        def _detect_action_from_query(q: str) -> Optional[str]:
-            qlow = (q or "").lower()
-            for act, syns in ACTION_SYNONYMS_EXT.items():
-                if any(s in qlow for s in syns):
-                    return act
-            return None
-
-        def _strip_boilerplate(raw_context: str) -> str:
-            """Remove document title/date/author/change-history noise from steps."""
-            MONTH_TERMS = ("january", "february", "march", "april", "may", "june",
-                           "july", "august", "september", "october", "november", "december")
-            lines = _normalize_lines(raw_context)
-            cleaned: List[str] = []
-            for ln in lines:
-                low = ln.lower()
-                is_change_hist = ("change history" in low) or ("initial draft" in low) or ("review" in low) or ("version" in low)
-                has_month_year = any(m in low for m in MONTH_TERMS) and bool(re.search(r"\b20\d{2}\b", low))
-                is_title_line = ("sop document" in low) or ("contents" in low)
-                if is_change_hist or has_month_year or is_title_line:
-                    continue
-                cleaned.append(ln)
-            return "\n".join(cleaned).strip()
-
-        def _extract_action_block(raw_context: str, target_act: Optional[str]) -> str:
-            """
-            Extract the contiguous block of lines for the target action (create/update/delete).
-            Start when a line mentions the target action OR looks procedural for 'create',
-            and stop ONLY when a line is a clear boundary:
-              - inline heading for another topic (e.g., 'Appointment schedule updation', 'Appointment deletion'), OR
-              - a line that strongly signals a different action (update/delete) via extended synonyms.
-            """
-            if not raw_context.strip():
-                return raw_context
-
-            lines = _normalize_lines(raw_context)
-            if not lines or not target_act:
-                return raw_context
-
-            INLINE_BOUNDARIES = (
-                "appointment schedule updation",
-                "schedule updation",
-                "appointment deletion",
-                "deletion",
-            )
-
-            other_terms: List[str] = []
-            for act, syns in ACTION_SYNONYMS_EXT.items():
-                if act != target_act:
-                    other_terms.extend(syns)
-            other_terms_low = set(t.lower() for t in other_terms)
-
-            def is_boundary_line(low: str) -> bool:
-                if any(h in low for h in INLINE_BOUNDARIES):
-                    return True
-                if any(t in low for t in other_terms_low):
-                    return True
-                return False
-
-            PROCEDURAL_VERBS = ("select", "choose", "click", "open", "add", "assign", "save",
-                                "navigate", "tag", "displayed", "triggered")
-            def is_procedural(low: str) -> bool:
-                return any(v in low for v in PROCEDURAL_VERBS)
-
-            target_terms_low = set(t.lower() for t in ACTION_SYNONYMS_EXT.get(target_act, []))
-
-            started = False
-            block: List[str] = []
-
-            for ln in lines:
-                low = ln.lower()
-
-                contains_target = any(t in low for t in target_terms_low)
-                if not started:
-                    if contains_target or (target_act == "create" and is_procedural(low)):
-                        started = True
-                        block.append(ln)
-                    continue
-
-                if is_boundary_line(low):
-                    break
-
-                block.append(ln)
-
-            return "\n".join(block).strip() if block else raw_context
-
-        def _filter_steps_by_action(raw_context: str, asked_act: Optional[str]) -> str:
-            cleaned = _strip_boilerplate(raw_context)
-            block = _extract_action_block(cleaned, asked_act)
-            if asked_act:
-                other_terms: List[str] = []
-                for act, syns in ACTION_SYNONYMS_EXT.items():
-                    if act != asked_act:
-                        other_terms.extend(syns)
-                lines = _normalize_lines(block)
-                lines = [ln for ln in lines if not any(t in ln.lower() for t in other_terms)]
-                block = "\n".join(lines).strip() if lines else block
-            return block
-
-        escalation_line = None
-        full_errors = None
+        # Build SOP context if allowed
+        if is_perm_query:
+            detected_intent = "errors"
+
+        escalation_line = None  # SOP escalation candidate
+        full_errors = None      # keep for possible escalation extraction
         next_step_applied = False
         next_step_info: Dict[str, Any] = {}
 
-        if best_doc and detected_intent == "steps":
-            context_preformatted = False
-            full_steps = None
-
-            # 1) Try by KB action tags
-            action_steps = _get_steps_for_action(best_doc, kb_results.get("actions", []))
-            if action_steps:
-                full_steps = action_steps
-            else:
-                if kb_results.get("actions"):
-                   alt_doc, alt_steps = _get_steps_for_action_global(kb_results["actions"], prefer_doc=best_doc)
-                   if alt_steps:
-                      best_doc = alt_doc   # switch to the doc that actually has the section
-                      full_steps = alt_steps
-                       
-                asked_action = _detect_action_from_query(input_data.user_message)
-                if not kb_results.get("actions") and asked_action:
-                   alt_doc, alt_steps = _get_steps_for_action_global([asked_action], prefer_doc=best_doc)
-                   if alt_steps:
-                      best_doc = alt_doc   # switch to the doc that actually has the section
-                      full_steps = alt_steps
-
-                if not full_steps:
-                   default_sec = _pick_default_action_section(best_doc)
-                   if default_sec:
-                      full_steps = get_section_text(best_doc, default_sec)
-                if not full_steps:
-                   full_steps = get_best_steps_section_text(best_doc)
+        if best_doc:
+            if detected_intent == "steps":
+                full_steps = get_best_steps_section_text(best_doc)
                 if not full_steps:
-                   sec = (top_meta or {}).get("section")
-                   if sec:
-                      full_steps = get_section_text(best_doc, sec)
-    
-            if full_steps:
-                # Always add Save lines if present anywhere in the doc (independent of query wording)
-                save_lines = _find_save_lines_in_doc(best_doc, max_lines=2)
-                if save_lines:
-                    low_steps = (full_steps or "").lower()
-                    if not any(s in low_steps for s in SAVE_SYNS):
-                        full_steps = (full_steps or "").rstrip() + "\n" + save_lines
-
-                asked_action = _detect_action_from_query(input_data.user_message)
-                full_steps = _filter_steps_by_action(full_steps, asked_action)
-
-                numbered_full = _ensure_numbering(full_steps)
-                next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
-
-                if next_only is not None:
-                    if len(next_only) == 0:
-                        context = "You are at the final step of this SOP. No further steps."
-                        next_step_applied = True
-                        next_step_info = {"count": 0}
-                        context_preformatted = True
+                    sec = (top_meta or {}).get("section")
+                    if sec:
+                        full_steps = get_section_text(best_doc, sec)
+
+                if full_steps:
+                    # Use numbered form only for matching; keep raw for full output
+                    numbered_full = _ensure_numbering(full_steps)
+                    next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
+
+                    if next_only is not None:
+                        # "what's next" mode
+                        if len(next_only) == 0:
+                            context = "You are at the final step of this SOP. No further steps."
+                            next_step_applied = True
+                            next_step_info = {"count": 0}
+                            context_preformatted = True
+                        else:
+                            context = _format_steps_as_numbered(next_only)
+                            next_step_applied = True
+                            next_step_info = {"count": len(next_only)}
+                            context_preformatted = True
                     else:
-                        context = _format_steps_as_numbered(next_only)
-                        next_step_applied = True
-                        next_step_info = {"count": len(next_only)}
-                        context_preformatted = True
-                else:
-                    context = full_steps
-                    context_preformatted = False
-
-                # Clear filter info for debug clarity
-                filt_info = {'mode': None, 'matched_count': None, 'all_sentences': None}
-                context_found = True
-
-        elif best_doc and detected_intent == "errors":
-            full_errors = get_best_errors_section_text(best_doc)
-            if full_errors:
-                ctx_err = _extract_errors_only(full_errors, max_lines=30)
-                if is_perm_query:
-                    context = _filter_permission_lines(ctx_err, max_lines=6)
-                else:
-                    is_specific_error = len(_detect_error_families(msg_low)) > 0
-                    if is_specific_error:
-                        context = _filter_error_lines_by_query(ctx_err, input_data.user_message, max_lines=1)
+                        # Normal mode: return the full SOP section (raw),
+                        # and we'll number it below once.
+                        context = full_steps
+                        context_preformatted = False
+
+            elif detected_intent == "errors":
+                full_errors = get_best_errors_section_text(best_doc)
+                if full_errors:
+                    ctx_err = _extract_errors_only(full_errors, max_lines=30)
+                    if is_perm_query:
+                        context = _filter_permission_lines(ctx_err, max_lines=6)
                     else:
-                        all_lines: List[str] = _normalize_lines(ctx_err)
-                        error_bullets = [ln for ln in all_lines if re.match(r"^\s*[-*\u2022]\s*", ln) or (":" in ln)]
-                        context = "\n".join(error_bullets[:8]).strip()
-                        assist_followup = (
-                            "Please tell me which error above matches your screen (paste the exact text), "
-                            "or share a screenshot. I can guide you further or raise a ServiceNow ticket."
-                        )
-                escalation_line = _extract_escalation_line(full_errors)
-            else:
-                full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
-                if full_steps:
-                    asked_action = _detect_action_from_query(input_data.user_message)
-                    full_steps = _filter_steps_by_action(full_steps, asked_action)
-                    context = full_steps
-                    detected_intent = "steps"
-                    context_preformatted = False
-
-        elif best_doc and detected_intent == "prereqs":
-            full_prereqs = _find_prereq_section_text(best_doc)
-            if full_prereqs:
-                context = full_prereqs.strip()
-            else:
-                full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
-                if full_steps:
-                    asked_action = _detect_action_from_query(input_data.user_message)
-                    full_steps = _filter_steps_by_action(full_steps, asked_action)
-                    context = full_steps
-                    detected_intent = "steps"
-                    context_preformatted = False
-
-        else:
-            # Neutral or other intents: use filtered context
-            context = filtered_context
+                        # Decide specific vs generic:
+                        is_specific_error = len(_detect_error_families(msg_low)) > 0
+                        if is_specific_error:
+                            context = _filter_error_lines_by_query(ctx_err, input_data.user_message, max_lines=1)
+                        else:
+                            all_lines: List[str] = _normalize_lines(ctx_err)
+                            error_bullets = [ln for ln in all_lines if re.match(r"^\s*[-*\u2022]\s*", ln) or (":" in ln)]
+                            context = "\n".join(error_bullets[:8]).strip()
+                            assist_followup = (
+                                "Please tell me which error above matches your screen (paste the exact text), "
+                                "or share a screenshot. I can guide you further or raise a ServiceNow ticket."
+                            )
+                    escalation_line = _extract_escalation_line(full_errors)
+
+            elif detected_intent == "prereqs":
+                full_prereqs = _find_prereq_section_text(best_doc)
+                if full_prereqs:
+                    context = full_prereqs.strip()
 
         language_hint = _detect_language_hint(input_data.user_message)
         lang_line = f"Respond in {language_hint}." if language_hint else "Respond in a clear, polite tone."
@@ -1391,6 +1116,8 @@ Return ONLY the rewritten guidance."""
 
         # Deterministic local formatting
         if detected_intent == "steps":
+            # If we trimmed to next steps, 'context' is already formatted (or a sentence).
+            # Only number when returning full SOP raw text.
             if ('context_preformatted' in locals()) and context_preformatted:
                 bot_text = context
             else:
@@ -1425,14 +1152,9 @@ Return ONLY the rewritten guidance."""
                     "Share a bit more detail (module/screen/error), or say ‘create ticket’."
                 )
 
-        # Status: mark OK when we served steps successfully
-        if detected_intent == "steps" and bot_text.strip():
-            status = "OK"
-        else:
-            short_query = len((input_data.user_message or "").split()) <= 4
-            gate_combined_ok = 0.60 if short_query else 0.55
-            status = "OK" if (best_combined is not None and best_combined >= gate_combined_ok) else "PARTIAL"
-
+        short_query = len((input_data.user_message or "").split()) <= 4
+        gate_combined_ok = 0.60 if short_query else 0.55
+        status = "OK" if (best_combined is not None and best_combined >= gate_combined_ok) else "PARTIAL"
         lower = (bot_text or "").lower()
         if ("partial" in lower) or ("may be partial" in lower) or ("closest" in lower) or ("may not fully" in lower):
             status = "PARTIAL"
@@ -1453,8 +1175,8 @@ Return ONLY the rewritten guidance."""
                 "best_distance": best_distance,
                 "best_combined": best_combined,
                 "http_status": http_code,
-                "filter_mode": (filt_info.get("mode") if filt_info else None),
-                "matched_count": (filt_info.get("matched_count") if filt_info else None),
+                "filter_mode": filt_info.get("mode"),
+                "matched_count": filt_info.get("matched_count"),
                 "user_intent": detected_intent,
                 "best_doc": best_doc,
                 "next_step": {