Update main.py

main.py

@@ -1,6 +1,4 @@
 
-# main_hugging_phase_recent.py
-
 import os
 import json
 import re
@@ -201,6 +199,52 @@ def _get_steps_for_action(best_doc: str, actions: list) -> Optional[str]:
             return txt.strip()
     return None
 
+
+# --- Default section picker when query doesn't reveal action ---
+def _pick_default_action_section(best_doc: str) -> Optional[str]:
+    """
+    If user actions are empty, prefer '...Creation' section,
+    else prefer '...Updation'/'...Update', else '...Deletion'/'...Cancel'.
+    Works generically for SOPs that use common headings.
+    """
+    order = ("creation", "updation", "update", "deletion", "delete", "cancel")
+    sections = []
+    for d in bm25_docs:
+        m = d.get("meta", {})
+        if m.get("filename") == best_doc and m.get("intent_tag") == "steps":
+            title = (m.get("section") or "").strip().lower()
+            if title:
+                sections.append(title)
+    for key in order:
+        for t in sections:
+            if key in t:
+                return t
+    return sections[0] if sections else None
+
+
+# --- Harvest 'Save' lines from ALL steps chunks in the doc (generic across SOPs) ---
+SAVE_SYNS = ("save", "saving", "save changes", "click save", "press save", "select save")
+
+def _find_save_lines_in_doc(best_doc: str, max_lines: int = 2) -> str:
+    """
+    Pulls up to max_lines lines that mention 'save' from any steps chunk in best_doc.
+    Returns a \n-joined string or empty if none found.
+    """
+    lines: List[str] = []
+    for d in bm25_docs:
+        m = d.get("meta", {})
+        if m.get("filename") != best_doc or m.get("intent_tag") != "steps":
+            continue
+        t = (d.get("text") or "").strip()
+        for ln in [x.strip() for x in t.splitlines() if x.strip()]:
+            low = ln.lower()
+            if any(s in low for s in SAVE_SYNS):
+                lines.append(ln)
+                if len(lines) >= max_lines:
+                    return "\n".join(lines)
+    return "\n".join(lines)
+
+
 def _is_domain_status_context(msg_norm: str) -> bool:
     if "status locked" in msg_norm or "locked status" in msg_norm:
         return True
@@ -943,8 +987,12 @@ async def chat_with_ai(input_data: ChatInput):
 
         selected = items[:max(1, 2)]
         context_raw = "\n\n---\n\n".join([s["text"] for s in selected]) if selected else ""
+
+        # Compute filter info for gating only; do NOT use the filtered text for steps
         filtered_text, filt_info = _filter_context_for_query(context_raw, input_data.user_message)
-        context = filtered_text
+        filtered_context = filtered_text
+
+        context = context_raw  # keep raw; we'll decide below
         context_found = bool(context.strip())
         best_distance = min([d for d in distances if d is not None], default=None) if distances else None
         best_combined = max([c for c in combined if c is not None], default=None) if combined else None
@@ -998,7 +1046,7 @@ async def chat_with_ai(input_data: ChatInput):
         if detected_intent in ("neutral", "prereqs") and looks_like_steps_query and looks_like_module:
             detected_intent = "steps"
 
-        # --- Meaning-aware SOP gating ---
+        # --- Meaning-aware SOP gating (uses filter info) ---
         def _contains_any(s: str, keywords: tuple) -> bool:
             low = (s or "").lower()
             return any(k in low for k in keywords)
@@ -1010,9 +1058,9 @@ async def chat_with_ai(input_data: ChatInput):
             "asn", "grn", "pick", "picking"
         )
         ACTION_OR_ERROR_TERMS = (
-            "how to", "procedure", "perform",  # added
+            "how to", "procedure", "perform",
             "close", "closing", "open", "navigate", "scan", "confirm", "generate", "update",
-            "receive", "receiving",            # added
+            "receive", "receiving",
             "error", "issue", "fail", "failed", "not working", "locked", "mismatch",
             "access", "permission", "status"
         )
@@ -1058,10 +1106,7 @@ async def chat_with_ai(input_data: ChatInput):
                 },
             }
 
-        # ---------- Action-focused extraction + boilerplate cleanup ----------
-        MONTH_TERMS = ("january", "february", "march", "april", "may", "june",
-                       "july", "august", "september", "october", "november", "december")
-
+        # ---------- Build SOP context ----------
         def _detect_action_from_query(q: str) -> Optional[str]:
             qlow = (q or "").lower()
             for act, syns in ACTION_SYNONYMS_EXT.items():
@@ -1071,6 +1116,8 @@ async def chat_with_ai(input_data: ChatInput):
 
         def _strip_boilerplate(raw_context: str) -> str:
             """Remove document title/date/author/change-history noise from steps."""
+            MONTH_TERMS = ("january", "february", "march", "april", "may", "june",
+                           "july", "august", "september", "october", "november", "december")
             lines = _normalize_lines(raw_context)
             cleaned: List[str] = []
             for ln in lines:
@@ -1158,91 +1205,110 @@ async def chat_with_ai(input_data: ChatInput):
                 block = "\n".join(lines).strip() if lines else block
             return block
 
-        # Build SOP context if allowed
-        if is_perm_query:
-            detected_intent = "errors"
-
         escalation_line = None
         full_errors = None
         next_step_applied = False
         next_step_info: Dict[str, Any] = {}
 
-        if best_doc:
-            if detected_intent == "steps":
-                action_steps = _get_steps_for_action(best_doc, kb_results.get("actions", []))
-                if action_steps:
-                   full_steps = action_steps
-                   context_preformatted = False  # we will number it below
-                    
-                else:
-        # existing fallback to the best steps bundle or the top section
+        if best_doc and detected_intent == "steps":
+            context_preformatted = False
+            full_steps = None
+
+            # 1) Try by KB action tags
+            action_steps = _get_steps_for_action(best_doc, kb_results.get("actions", []))
+            if action_steps:
+                full_steps = action_steps
+            else:
+                # 2) If no user action, pick a sensible default section (creation > update > deletion)
+                default_sec = _pick_default_action_section(best_doc)
+                if default_sec:
+                    full_steps = get_section_text(best_doc, default_sec)
+                # 3) Last resort: aggregated steps for the doc
+                if not full_steps:
                     full_steps = get_best_steps_section_text(best_doc)
-                    if not full_steps:
-                       sec = (top_meta or {}).get("section")
-                       if sec:
-                          full_steps = get_section_text(best_doc, sec)
+                # 4) Final fallback: top hit section
+                if not full_steps:
+                    sec = (top_meta or {}).get("section")
+                    if sec:
+                        full_steps = get_section_text(best_doc, sec)
+
+            if full_steps:
+                # Always add Save lines if present anywhere in the doc (independent of query wording)
+                save_lines = _find_save_lines_in_doc(best_doc, max_lines=2)
+                if save_lines:
+                    low_steps = (full_steps or "").lower()
+                    if not any(s in low_steps for s in SAVE_SYNS):
+                        full_steps = (full_steps or "").rstrip() + "\n" + save_lines
+
+                asked_action = _detect_action_from_query(input_data.user_message)
+                full_steps = _filter_steps_by_action(full_steps, asked_action)
+
+                numbered_full = _ensure_numbering(full_steps)
+                next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
+
+                if next_only is not None:
+                    if len(next_only) == 0:
+                        context = "You are at the final step of this SOP. No further steps."
+                        next_step_applied = True
+                        next_step_info = {"count": 0}
+                        context_preformatted = True
+                    else:
+                        context = _format_steps_as_numbered(next_only)
+                        next_step_applied = True
+                        next_step_info = {"count": len(next_only)}
+                        context_preformatted = True
+                else:
+                    context = full_steps
+                    context_preformatted = False
+
+                # Clear filter info for debug clarity
+                filt_info = {'mode': None, 'matched_count': None, 'all_sentences': None}
+                context_found = True
+
+        elif best_doc and detected_intent == "errors":
+            full_errors = get_best_errors_section_text(best_doc)
+            if full_errors:
+                ctx_err = _extract_errors_only(full_errors, max_lines=30)
+                if is_perm_query:
+                    context = _filter_permission_lines(ctx_err, max_lines=6)
+                else:
+                    is_specific_error = len(_detect_error_families(msg_low)) > 0
+                    if is_specific_error:
+                        context = _filter_error_lines_by_query(ctx_err, input_data.user_message, max_lines=1)
+                    else:
+                        all_lines: List[str] = _normalize_lines(ctx_err)
+                        error_bullets = [ln for ln in all_lines if re.match(r"^\s*[-*\u2022]\s*", ln) or (":" in ln)]
+                        context = "\n".join(error_bullets[:8]).strip()
+                        assist_followup = (
+                            "Please tell me which error above matches your screen (paste the exact text), "
+                            "or share a screenshot. I can guide you further or raise a ServiceNow ticket."
+                        )
+                escalation_line = _extract_escalation_line(full_errors)
+            else:
+                full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
                 if full_steps:
                     asked_action = _detect_action_from_query(input_data.user_message)
                     full_steps = _filter_steps_by_action(full_steps, asked_action)
+                    context = full_steps
+                    detected_intent = "steps"
+                    context_preformatted = False
+
+        elif best_doc and detected_intent == "prereqs":
+            full_prereqs = _find_prereq_section_text(best_doc)
+            if full_prereqs:
+                context = full_prereqs.strip()
+            else:
+                full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
+                if full_steps:
+                    asked_action = _detect_action_from_query(input_data.user_message)
+                    full_steps = _filter_steps_by_action(full_steps, asked_action)
+                    context = full_steps
+                    detected_intent = "steps"
+                    context_preformatted = False
 
-                    numbered_full = _ensure_numbering(full_steps)
-                    next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
-
-                    if next_only is not None:
-                        if len(next_only) == 0:
-                            context = "You are at the final step of this SOP. No further steps."
-                            next_step_applied = True
-                            next_step_info = {"count": 0}
-                            context_preformatted = True
-                        else:
-                            context = _format_steps_as_numbered(next_only)
-                            next_step_applied = True
-                            next_step_info = {"count": len(next_only)}
-                            context_preformatted = True
-                    else:
-                        context = full_steps
-                        context_preformatted = False
-
-            elif detected_intent == "errors":
-                full_errors = get_best_errors_section_text(best_doc)
-                if full_errors:
-                    ctx_err = _extract_errors_only(full_errors, max_lines=30)
-                    if is_perm_query:
-                        context = _filter_permission_lines(ctx_err, max_lines=6)
-                    else:
-                        is_specific_error = len(_detect_error_families(msg_low)) > 0
-                        if is_specific_error:
-                            context = _filter_error_lines_by_query(ctx_err, input_data.user_message, max_lines=1)
-                        else:
-                            all_lines: List[str] = _normalize_lines(ctx_err)
-                            error_bullets = [ln for ln in all_lines if re.match(r"^\s*[-*\u2022]\s*", ln) or (":" in ln)]
-                            context = "\n".join(error_bullets[:8]).strip()
-                            assist_followup = (
-                                "Please tell me which error above matches your screen (paste the exact text), "
-                                "or share a screenshot. I can guide you further or raise a ServiceNow ticket."
-                            )
-                    escalation_line = _extract_escalation_line(full_errors)
-                else:
-                    full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
-                    if full_steps:
-                        asked_action = _detect_action_from_query(input_data.user_message)
-                        full_steps = _filter_steps_by_action(full_steps, asked_action)
-                        context = full_steps
-                        detected_intent = "steps"
-                        context_preformatted = False
-
-            elif detected_intent == "prereqs":
-                full_prereqs = _find_prereq_section_text(best_doc)
-                if full_prereqs:
-                    context = full_prereqs.strip()
-                else:
-                    full_steps = get_best_steps_section_text(best_doc) or get_section_text(best_doc, sec_title or "")
-                    if full_steps:
-                        asked_action = _detect_action_from_query(input_data.user_message)
-                        full_steps = _filter_steps_by_action(full_steps, asked_action)
-                        context = full_steps
-                        detected_intent = "steps"
-                        context_preformatted = False
+        else:
+            # Neutral or other intents: use filtered context
+            context = filtered_context
 
         language_hint = _detect_language_hint(input_data.user_message)
         lang_line = f"Respond in {language_hint}." if language_hint else "Respond in a clear, polite tone."
@@ -1312,9 +1378,14 @@ Return ONLY the rewritten guidance."""
                     "Share a bit more detail (module/screen/error), or say ‘create ticket’."
                 )
 
-        short_query = len((input_data.user_message or "").split()) <= 4
-        gate_combined_ok = 0.60 if short_query else 0.55
-        status = "OK" if (best_combined is not None and best_combined >= gate_combined_ok) else "PARTIAL"
+        # Status: mark OK when we served steps successfully
+        if detected_intent == "steps" and bot_text.strip():
+            status = "OK"
+        else:
+            short_query = len((input_data.user_message or "").split()) <= 4
+            gate_combined_ok = 0.60 if short_query else 0.55
+            status = "OK" if (best_combined is not None and best_combined >= gate_combined_ok) else "PARTIAL"
+
         lower = (bot_text or "").lower()
         if ("partial" in lower) or ("may be partial" in lower) or ("closest" in lower) or ("may not fully" in lower):
             status = "PARTIAL"
@@ -1335,8 +1406,8 @@ Return ONLY the rewritten guidance."""
                 "best_distance": best_distance,
                 "best_combined": best_combined,
                 "http_status": http_code,
-                "filter_mode": filt_info.get("mode"),
-                "matched_count": filt_info.get("matched_count"),
+                "filter_mode": (filt_info.get("mode") if filt_info else None),
+                "matched_count": (filt_info.get("matched_count") if filt_info else None),
                 "user_intent": detected_intent,
                 "best_doc": best_doc,
                 "next_step": {