Update main.py

main.py

@@ -315,9 +315,8 @@ def _soft_match_score(a: str, b: str) -> float:
 def _detect_next_intent(user_query: str) -> bool:
     q = _norm_text(user_query)
     keys = [
-        'after','after this','what next','whats next','next step',
-        'then what','following step','continue','subsequent','proceed',
-        'what to do','what should i do','how to proceed','how do i continue','proceed further','next?'
+        "after", "after this", "what next", "whats next", "next step",
+        "then what", "following step", "continue", "subsequent", "proceed"
     ]
     return any(k in q for k in keys)
 
@@ -327,41 +326,39 @@ def _resolve_next_steps(user_query: str, numbered_text: str, max_next: int = 8,
     1) Detect 'what next' intent.
     2) Stem & match query against each step using tokens + bigrams + synonyms.
     3) If a good anchor is found, return ONLY subsequent steps (window=max_next).
-    Else return None (fallback to full SOP rendering).
+       Else return None (fallback to full SOP rendering).
     """
     if not _detect_next_intent(user_query):
         return None
+
     steps = _split_sop_into_steps(numbered_text)
     if not steps:
         return None
+
     q_norm = _norm_text(user_query)
     q_tokens = [t for t in q_norm.split() if len(t) > 1]
+
     best_idx, best_score = -1, -1.0
     for idx, step in enumerate(steps):
+        # base fuzzy score
         s1 = _soft_match_score(user_query, step)
+        # synonym hits
         syn = _syn_hits(q_tokens, step)
+        # combined score (synonyms are discrete)
         score = s1 + 0.12 * syn
         if score > best_score:
             best_score, best_idx = score, idx
+
+    # Looser threshold to accept anchors with synonyms / tense differences
     if best_idx < 0 or best_score < min_score:
-        return None
+        return None  # let caller fall back to the full SOP
+
     start = best_idx + 1
     if start >= len(steps):
-        return []
+        return []  # already at final step
+
     end = min(start + max_next, len(steps))
-    anchor_norm = _norm_text(steps[best_idx])
-    def _unique(seq):
-        seen = set()
-        out = []
-        for s in seq:
-            k = _norm_text(s)
-            if k == anchor_norm:
-                continue
-            if k not in seen:
-                seen.add(k)
-                out.append(s)
-        return out
-    return _unique(steps[start:end])
+    return steps[start:end]
 
 def _syn_hits(q_tokens: List[str], step_line: str) -> int:
     """
@@ -631,12 +628,153 @@ async def health_check():
 # ------------------------------------------------------------------------------
 # Chat
 # ------------------------------------------------------------------------------
+
+
+# --- Helpers for section slicing and action-filter ---
+def _slice_steps_by_known_headings(text: str, actions: List[str]) -> str:
+    """
+    If the SOP mixes creation/update/deletion under one 'Document' section,
+    carve out the subsection by looking for heading markers in the text.
+    """
+    if not text:
+        return text
+    act = (actions or [None])[0]
+    lines = [ln for ln in (text or '').splitlines() if ln.strip()]
+    if not lines or not act:
+        return text
+    act = act.lower()
+    idx_create = idx_update = idx_delete = None
+    for i, ln in enumerate(lines):
+        low = ln.lower()
+        if idx_create is None and ('appointment' in low and 'creat' in low and 'step' in low):
+            idx_create = i
+        if idx_update is None and ('appointment' in low and 'updat' in low and 'step' in low):
+            idx_update = i
+        if idx_delete is None and (('deletion' in low) or ('delete' in low)) and ('appointment' in low and 'step' in low):
+            idx_delete = i
+    start = None
+    if act == 'create':
+        start = idx_create
+    elif act == 'update':
+        start = idx_update
+    elif act == 'delete':
+        start = idx_delete
+    if start is None:
+        return text
+    next_idxs = sorted([x for x in (idx_create, idx_update, idx_delete) if x is not None and x > start])
+    end = next_idxs[0] if next_idxs else len(lines)
+    sub = '\n'.join(lines[start:end]).strip()
+    return sub or text
+
+
+def _filter_steps_for_action(text: str, actions: List[str]) -> str:
+    """Remove lines that conflict with requested action (create/update/delete)."""
+    if not text:
+        return text
+    lines = [ln for ln in (text or '').splitlines() if ln.strip()]
+    if not lines or not actions:
+        return text
+    act = (actions or [None])[0]
+    if not act:
+        return text
+    act = act.lower()
+    conflicts = {
+        'create': ('delete','remove','update','modify','change','edit'),
+        'update': ('delete','remove','create','add','new','generate'),
+        'delete': ('create','add','new','generate','update','modify','change','edit'),
+    }
+    conf = conflicts.get(act, ())
+    kept = [ln for ln in lines if not any(c in ln.lower() for c in conf)]
+    return '\n'.join(kept).strip() if kept else text
+
+# --- Overrides with safer logic ---
+def _detect_next_intent(user_query: str) -> bool:
+    q = _norm_text(user_query)
+    keys = [
+        'after','after this','what next','whats next','next step',
+        'then what','following step','continue','subsequent','proceed',
+        'what to do','what should i do','how to proceed','how do i continue','proceed further','next?'
+    ]
+    return any(k in q for k in keys)
+
+
+def _resolve_next_steps(user_query: str, numbered_text: str, max_next: int = 8, min_score: float = 0.25):
+    """
+    Detect 'what next' intent and return only subsequent steps.
+    Excludes the anchor step and de-duplicates the result.
+    """
+    if not _detect_next_intent(user_query):
+        return None
+    steps = _split_sop_into_steps(numbered_text)
+    if not steps:
+        return None
+    q_norm = _norm_text(user_query)
+    q_tokens = [t for t in q_norm.split() if len(t) > 1]
+    best_idx, best_score = -1, -1.0
+    for idx, step in enumerate(steps):
+        s1 = _soft_match_score(user_query, step)
+        syn = _syn_hits(q_tokens, step)
+        score = s1 + 0.12 * syn
+        if score > best_score:
+            best_score, best_idx = score, idx
+    if best_idx < 0 or best_score < min_score:
+        return None
+    start = best_idx + 1
+    if start >= len(steps):
+        return []
+    end = min(start + max_next, len(steps))
+    anchor_norm = _norm_text(steps[best_idx])
+    out, seen = [], set()
+    for s in steps[start:end]:
+        k = _norm_text(s)
+        if k == anchor_norm:
+            continue
+        if k not in seen:
+            seen.add(k)
+            out.append(s)
+    return out
+
+
+def _syn_hits(q_tokens: List[str], step_line: str) -> int:
+    """Synonym/alias hits tailored for WMS picking + appointments wording."""
+    step = _norm_text(step_line)
+    SYNS = {
+        'scan': {'scan', 'prompt', 'display', 'show'},
+        'confirm': {'confirm', 'verify', 'check'},
+        'item': {'item', 'sku', 'product'},
+        'qty': {'qty', 'quantity', 'count'},
+        'place': {'place', 'put', 'stow'},
+        'complete': {'complete', 'finish', 'done'},
+        'staging': {'staging', 'stage', 'dock', 'door'},
+        'location': {'location', 'loc'},
+        'pallet': {'pallet', 'container'},
+        # Appointment vocabulary
+        'appointment': {'appointment', 'slot', 'schedule'},
+        'assign': {'assign', 'tag'},
+        'save': {'save', 'saved'},
+        'delete': {'delete', 'remove', 'removed'},
+        'update': {'update', 'modify', 'change', 'edit', 'updated'},
+        'vendor': {'vendor', 'supplier'},
+        'pro': {'pro number', 'pro'},
+    }
+    hits = 0
+    for qt in q_tokens:
+        for fam, words in SYNS.items():
+            if qt == fam or qt in words:
+                if any(w in step for w in words):
+                    hits += 1
+    PHRASES = ('scan location', 'confirm item', 'pick quantity', 'place picked', 'move pallet', 'staging area')
+    for ph in PHRASES:
+        if ph in step:
+            hits += 1
+    return hits
+
+
 @app.post("/chat")
 async def chat_with_ai(input_data: ChatInput):
     assist_followup: Optional[str] = None
     try:
         msg_norm = (input_data.user_message or "").lower().strip()
-
         # Yes/No handlers
         if msg_norm in ("yes", "y", "sure", "ok", "okay"):
             return {
@@ -660,7 +798,6 @@ async def chat_with_ai(input_data: ChatInput):
         is_llm_resolved = _classify_resolution_llm(input_data.user_message)
         if _has_negation_resolved(msg_norm):
             is_llm_resolved = False
-
         if (not _has_negation_resolved(msg_norm)) and (_is_resolution_ack_heuristic(msg_norm) or is_llm_resolved):
             try:
                 short_desc, long_desc = _build_tracking_descriptions(input_data.last_issue, input_data.user_message)
@@ -731,7 +868,7 @@ async def chat_with_ai(input_data: ChatInput):
                 "debug": {"intent": "create_ticket"},
             }
 
-        # Status intent (ticket/incident) — disambiguated
+        # Status intent (ticket/incident)
         status_intent = _parse_ticket_status_intent(msg_norm)
         if status_intent:
             if status_intent.get("ask_number"):
@@ -755,7 +892,6 @@ async def chat_with_ai(input_data: ChatInput):
                 instance_url = os.getenv("SERVICENOW_INSTANCE_URL")
                 if not instance_url:
                     raise HTTPException(status_code=500, detail="SERVICENOW_INSTANCE_URL missing")
-
                 headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
                 number = status_intent.get("number")
                 url = f"{instance_url}/api/now/table/incident?number={number}"
@@ -763,12 +899,10 @@ async def chat_with_ai(input_data: ChatInput):
                 data = response.json()
                 lst = data.get("result", [])
                 result = (lst or [{}])[0] if response.status_code == 200 else {}
-
                 state_code = builtins.str(result.get("state", "unknown"))
                 state_label = STATE_MAP.get(state_code, state_code)
                 short = result.get("short_description", "")
                 num = result.get("number", number or "unknown")
-
                 return {
                     "bot_response": (
                         f"**Ticket:** {num}\n"
@@ -788,9 +922,7 @@ async def chat_with_ai(input_data: ChatInput):
             except Exception as e:
                 raise HTTPException(status_code=500, detail=safe_str(e))
 
-        # -----------------------------
         # Hybrid KB search
-        # -----------------------------
         kb_results = hybrid_search_knowledge_base(input_data.user_message, top_k=10, alpha=0.6, beta=0.4)
         documents = kb_results.get("documents", [])
         metadatas = kb_results.get("metadatas", [])
@@ -814,141 +946,44 @@ async def chat_with_ai(input_data: ChatInput):
 
         selected = items[:max(1, 2)]
         context_raw = "\n\n---\n\n".join([s["text"] for s in selected]) if selected else ""
-
         filtered_text, filt_info = _filter_context_for_query(context_raw, input_data.user_message)
         context = filtered_text
         context_found = bool(context.strip())
 
-        best_distance = (
-            min([d for d in distances if d is not None], default=None) if distances else None
-        )
-        best_combined = (
-            max([c for c in combined if c is not None], default=None) if combined else None
-        )
-
-        detected_intent = kb_results.get("user_intent", "neutral")
+        best_combined = (max([c for c in combined if c is not None], default=None) if combined else None)
         best_doc = kb_results.get("best_doc")
         top_meta = (metadatas or [{}])[0] if metadatas else {}
-        msg_low = (input_data.user_message or "").lower()
 
+        msg_low = (input_data.user_message or "").lower()
         GENERIC_ERROR_TERMS = ("error", "issue", "problem", "not working", "failed", "failure")
         generic_error_signal = any(t in msg_low for t in GENERIC_ERROR_TERMS)
 
-        # Query-based prereq nudge
-        PREREQ_TERMS = (
-            "pre req", "pre-requisite", "pre-requisites", "prerequisite",
-            "prerequisites", "pre requirement", "pre-requirements", "requirements"
-        )
-        if detected_intent == "neutral" and any(t in msg_low for t in PREREQ_TERMS):
-            detected_intent = "prereqs"
-
-        # Permissions force
-        PERM_QUERY_TERMS = [
-            "permission", "permissions", "access", "access right", "authorization", "authorisation",
-            "role", "role access", "security", "security profile", "privilege",
-            "not allowed", "not authorized", "denied",
-        ]
-        is_perm_query = any(t in msg_norm for t in PERM_QUERY_TERMS)
-        if is_perm_query:
-            detected_intent = "errors"
-
-        # Heading-aware prereq nudge
-        sec_title = ((top_meta or {}).get("section") or "").strip().lower()
-        PREREQ_HEADINGS = (
-            "pre-requisites", "prerequisites", "pre requisites",
-            "pre-requirements", "requirements"
-        )
-        if detected_intent == "neutral" and any(h in sec_title for h in PREREQ_HEADINGS):
-            detected_intent = "prereqs"
-
-        # ---- FORCE STEPS for "what's next" / "next step" queries ----
+        # Force steps on next-intent
         try:
             if _detect_next_intent(input_data.user_message):
                 detected_intent = "steps"
+            else:
+                detected_intent = kb_results.get("user_intent", "neutral")
         except Exception:
-            pass
-
-        # Gating
-        def _contains_any(s: str, keywords: tuple) -> bool:
-            low = (s or "").lower()
-            return any(k in low for k in keywords)
-
-        DOMAIN_TERMS = (
-            "trailer", "shipment", "order", "load", "wave",
-            "inventory", "putaway", "receiving", "appointment",
-            "dock", "door", "manifest", "pallet", "container",
-            "asn", "grn", "pick", "picking"
-        )
-        ACTION_OR_ERROR_TERMS = (
-            "how to", "procedure", "perform",
-            "close", "closing", "open", "navigate", "scan", "confirm", "generate", "update",
-            "receive", "receiving",
-            "error", "issue", "fail", "failed", "not working", "locked", "mismatch",
-            "access", "permission", "status"
-        )
+            detected_intent = kb_results.get("user_intent", "neutral")
 
-        matched_count = int(filt_info.get("matched_count") or 0)
-        filter_mode = (filt_info.get("mode") or "").lower()
-        has_any_action_or_error = _contains_any(msg_low, ACTION_OR_ERROR_TERMS)
-        mentions_domain = _contains_any(msg_low, DOMAIN_TERMS)
-
-        short_query = len((input_data.user_message or "").split()) <= 4
-        gate_combined_ok = 0.60 if short_query else 0.55
-        combined_ok = (best_combined is not None and best_combined >= gate_combined_ok)
-        weak_domain_only = (mentions_domain and not has_any_action_or_error)
-        low_context_hit = (matched_count < 2 and filter_mode in ("concise", "exact"))
-
-        strong_steps_bypass = True  # next-step override already set steps; allow
-        strong_error_signal = len(_detect_error_families(msg_low)) > 0
-
-        if (weak_domain_only or (low_context_hit and not combined_ok)) \
-           and not strong_steps_bypass \
-           and not (strong_error_signal or generic_error_signal):
-            return {
-                "bot_response": _build_clarifying_message(),
-                "status": "NO_KB_MATCH",
-                "context_found": False,
-                "ask_resolved": False,
-                "suggest_incident": True,
-                "followup": "Share more details (module/screen/error), or say 'create ticket'.",
-                "options": [{"type": "yesno", "title": "Share details or raise a ticket?"}],
-                "top_hits": [],
-                "sources": [],
-                "debug": {
-                    "intent": "sop_rejected_weak_match",
-                    "matched_count": matched_count,
-                    "filter_mode": filter_mode,
-                    "best_combined": best_combined,
-                    "mentions_domain": mentions_domain,
-                    "has_any_action_or_error": has_any_action_or_error,
-                    "strong_steps_bypass": strong_steps_bypass,
-                    "strong_error_signal": strong_error_signal,
-                    "generic_error_signal": generic_error_signal,
-                },
-            }
-
-        # Build SOP context if allowed
         escalation_line: Optional[str] = None
-        full_errors: Optional[str] = None
         next_step_applied = False
         next_step_info: Dict[str, Any] = {}
         context_preformatted = False
 
         if best_doc and detected_intent == "steps":
-            full_steps = get_best_steps_section_text(best_doc)
-            if not full_steps:
-                sec = (top_meta or {}).get("section")
-                if sec:
-                    full_steps = get_section_text(best_doc, sec)
-
+            sec = (top_meta or {}).get('section')
+            if sec:
+                full_steps = get_section_text(best_doc, sec)
+            else:
+                full_steps = get_best_steps_section_text(best_doc)
             if full_steps:
-                numbered_full = _ensure_numbering(full_steps)
-                next_only = _resolve_next_steps(
-                    input_data.user_message,
-                    numbered_full,
-                    max_next=6,
-                    min_score=0.35
-                )
+                actions = kb_results.get('actions', [])
+                sliced = _slice_steps_by_known_headings(full_steps, actions)
+                filtered = _filter_steps_for_action(sliced, actions) if actions else sliced
+                numbered_full = _ensure_numbering(filtered)
+                next_only = _resolve_next_steps(input_data.user_message, numbered_full, max_next=6, min_score=0.35)
                 if next_only is not None:
                     if len(next_only) == 0:
                         context = "You are at the final step of this SOP. No further steps."
@@ -961,158 +996,42 @@ async def chat_with_ai(input_data: ChatInput):
                         next_step_info = {"count": len(next_only)}
                         context_preformatted = True
                 else:
-                    context = full_steps
+                    context = filtered
                     context_preformatted = False
-
-                # clear filter info for debug clarity
-                filt_info = {'mode': None, 'matched_count': None, 'all_sentences': None}
-                context_found = True
-
-        elif best_doc and detected_intent == "errors":
-            full_errors = get_best_errors_section_text(best_doc)
-            if full_errors:
-                ctx_err = _extract_errors_only(full_errors, max_lines=30)
-                if is_perm_query:
-                    context = _filter_permission_lines(ctx_err, max_lines=6)
-                else:
-                    is_specific_error = len(_detect_error_families(msg_low)) > 0
-                    if is_specific_error:
-                        context = _filter_context_for_query(ctx_err, input_data.user_message)[0]
-                    else:
-                        all_lines: List[str] = _normalize_lines(ctx_err)
-                        error_bullets = [
-                            ln for ln in all_lines
-                            if re.match(r"^\s*[\-\*\u2022]\s*", ln) or (":" in ln)
-                        ]
-                        context = "\n".join(error_bullets[:8]).strip()
-                        assist_followup = (
-                            "Please tell me which error above matches your screen (paste the exact text), "
-                            "or share a screenshot. I can guide you further or raise a ServiceNow ticket."
-                        )
-                escalation_line = _extract_escalation_line(full_errors)
-
-        elif best_doc and detected_intent == "prereqs":
-            full_prereqs = _find_prereq_section_text(best_doc)
-            if full_prereqs:
-                context = full_prereqs.strip()
                 context_found = True
 
         else:
-            # Neutral or other intents: use filtered context
+            # Non-steps intents fall back to filtered context
             context = filtered_text
 
-        # Language hint & paraphrase (for errors only)
-        language_hint = _detect_language_hint(input_data.user_message)
-        lang_line = f"Respond in {language_hint}." if language_hint else "Respond in a clear, polite tone."
-        use_gemini = (detected_intent == "errors")
-
-        enhanced_prompt = f"""You are a helpful support assistant. Rewrite the provided context ONLY into clear, user-friendly guidance.
-- Do not add any information that is not present in the context.
-- If the content is an error/access/permission note, paraphrase it into a helpful sentence users can understand.
-- {lang_line}
-### Context
-{context}
-### Question
-{input_data.user_message}
-### Output
-Return ONLY the rewritten guidance."""
-
-        headers = {"Content-Type": "application/json"}
-        payload = {"contents": [{"parts": [{"text": enhanced_prompt}]}]}
-        bot_text = ""
-        http_code = 0
-
-        if use_gemini and GEMINI_API_KEY:
-            try:
-                resp = requests.post(GEMINI_URL, headers=headers, json=payload, timeout=25, verify=GEMINI_SSL_VERIFY)
-                http_code = getattr(resp, "status_code", 0)
-                try:
-                    result = resp.json()
-                except Exception:
-                    result = {}
-                try:
-                    bot_text = result.get("candidates", [{}])[0].get("content", {}).get("parts", [{}])[0].get("text", "")
-                except Exception:
-                    bot_text = ""
-            except Exception:
-                bot_text, http_code = "", 0
-
-        # Deterministic local formatting
+        # Final formatting
         if detected_intent == "steps":
-            if context_preformatted:
-                bot_text = context
-            else:
-                bot_text = _ensure_numbering(context)
-        elif detected_intent == "errors":
-            if not (bot_text or "").strip() or http_code == 429:
-                bot_text = context.strip()
-            if escalation_line:
-                bot_text = (bot_text or "").rstrip() + "\n\n" + escalation_line
+            bot_text = context if context_preformatted else _ensure_numbering(context)
         else:
             bot_text = context
 
-        # Append escalation if explicitly requested even in steps mode
-        needs_escalation = (" escalate" in msg_norm) or ("escalation" in msg_norm)
-        if needs_escalation and best_doc:
-            esc_text = get_escalation_text(best_doc)
-            if not esc_text and full_errors:
-                esc_text = full_errors
-            line = _extract_escalation_line(esc_text or "")
-            if line:
-                bot_text = (bot_text or "").rstrip() + "\n\n" + line
-
-        # Non-empty guarantee
-        if not (bot_text or "").strip():
-            if context.strip():
-                bot_text = context.strip()
-            else:
-                bot_text = (
-                    "I found some related guidance but couldn’t assemble a reply. "
-                    "Share a bit more detail (module/screen/error), or say ‘create ticket’."
-                )
-
-        short_query = len((input_data.user_message or "").split()) <= 4
-        gate_combined_ok = 0.60 if short_query else 0.55
-        status = "OK" if (best_combined is not None and best_combined >= gate_combined_ok) else "PARTIAL"
-
-        lower = (bot_text or "").lower()
-        if ("partial" in lower) or ("may be partial" in lower) or ("closest" in lower) or ("may not fully" in lower):
-            status = "PARTIAL"
-
-        options = [{"type": "yesno", "title": "Share details or raise a ticket?"}] if status == "PARTIAL" else []
-
         return {
             "bot_response": bot_text,
-            "status": status,
+            "status": "OK" if (best_combined is not None and best_combined >= 0.55) else "PARTIAL",
             "context_found": context_found,
-            "ask_resolved": (status == "OK" and detected_intent != "errors"),
-            "suggest_incident": (status == "PARTIAL"),
-            "followup": (assist_followup if assist_followup else ("Is this helpful, or should I raise a ticket?" if status == "PARTIAL" else None)),
-            "options": options,
+            "ask_resolved": False,
+            "suggest_incident": False,
+            "followup": None,
+            "options": [],
             "top_hits": [],
             "sources": [],
             "debug": {
-                "used_chunks": len((context or "").split("\n\n---\n\n")) if context else 0,
-                "best_distance": best_distance,
-                "best_combined": best_combined,
-                "http_status": http_code,
-                "filter_mode": filt_info.get("mode") if isinstance(filt_info, dict) else None,
-                "matched_count": filt_info.get("matched_count") if isinstance(filt_info, dict) else None,
                 "user_intent": detected_intent,
                 "best_doc": best_doc,
                 "next_step": {"applied": next_step_applied, "info": next_step_info},
             },
         }
-
     except HTTPException:
         raise
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
 
-# ------------------------------------------------------------------------------
-# Ticket description generation
-# ------------------------------------------------------------------------------
 @app.post("/generate_ticket_desc")
 async def generate_ticket_desc_ep(input_data: TicketDescInput):
     try: