Update main.py

main.py

@@ -12,6 +12,7 @@ from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
 from dotenv import load_dotenv
 from datetime import datetime
+
 from services.kb_creation import (
     collection,
     ingest_documents,
@@ -26,15 +27,18 @@ from services.generate_ticket import get_valid_token, create_incident
 VERIFY_SSL = os.getenv("SERVICENOW_SSL_VERIFY", "true").lower() in ("1", "true", "yes")
 GEMINI_SSL_VERIFY = os.getenv("GEMINI_SSL_VERIFY", "true").lower() in ("1", "true", "yes")
 
+
 def safe_str(e: Any) -> str:
     try:
         return builtins.str(e)
     except Exception:
         return "<error stringify failed>"
 
+
 load_dotenv()
 os.environ["POSTHOG_DISABLED"] = "true"
 
+
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     try:
@@ -48,6 +52,7 @@ async def lifespan(app: FastAPI):
         print(f"[KB] ingestion failed: {safe_str(e)}")
     yield
 
+
 app = FastAPI(lifespan=lifespan)
 app.include_router(login_router)
 
@@ -60,23 +65,28 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+# ------------------ Models ------------------
 class ChatInput(BaseModel):
     user_message: str
     prev_status: Optional[str] = None
     last_issue: Optional[str] = None
 
+
 class IncidentInput(BaseModel):
     short_description: str
     description: str
     mark_resolved: Optional[bool] = False
 
+
 class TicketDescInput(BaseModel):
     issue: str
 
+
 class TicketStatusInput(BaseModel):
     sys_id: Optional[str] = None
     number: Optional[str] = None
 
+
 STATE_MAP = {
     "1": "New",
     "2": "In Progress",
@@ -92,27 +102,48 @@ GEMINI_URL = (
     f"gemini-2.5-flash-lite:generateContent?key={GEMINI_API_KEY}"
 )
 
-# ---------------- Numbering & friendly fallbacks ----------------
+# ------------------ Numbering & friendly fallbacks ------------------
 def _normalize_lines(text: str) -> List[str]:
-    return [ln.strip() for ln in (text or "").splitlines() if ln.strip()]
+    """
+    Safe splitter used everywhere to avoid undefined local variable bugs.
+    Always returns a list (possibly empty), never raises.
+    """
+    raw = (text or "")
+    try:
+        return [ln.strip() for ln in raw.splitlines() if ln.strip()]
+    except Exception:
+        return [raw.strip()] if raw.strip() else []
+
 
 def _ensure_numbering(text: str) -> str:
     """
     Force plain-text numbering: 1) 2) 3) so it renders in any bubble.
+    Handles bullets (-, *, •), and "Step N" formats. Splits single-paragraph procedures.
     """
-    lines = [ln.strip() for ln in (text or "").splitlines() if ln.strip()]
+    lines = _normalize_lines(text)
     if not lines:
         return text or ""
-    normalized = []
+
+    normalized: List[str] = []
     for ln in lines:
-        ln2 = re.sub(r"^\s*(?:\d+[\.\)]\s+|[\-\*\u2022]\s+)", "", ln).strip()
+        ln2 = re.sub(
+            r"^\s*(?:\d+[.\)\]]\s+|(?:step\s*\d+:?)\s+|[\-\*\u2022]\s+)",
+            "",
+            ln,
+            flags=re.IGNORECASE,
+        ).strip()
         normalized.append(ln2)
+
     if len(normalized) == 1:
-        parts = [p.strip() for p in re.split(r"\.\s+(?=[A-Z0-9])", normalized[0]) if p.strip()]
+        para = normalized[0]
+        # split on ';' or '. <Upper/Digit>' boundaries
+        parts = [p.strip() for p in re.split(r"(?:;|\.\s+(?=[A-Z0-9]))", para) if p.strip()]
         if len(parts) > 1:
             normalized = parts
+
     return "\n".join([f"{i+1}) {ln}" for i, ln in enumerate(normalized)])
 
+
 def _friendly_permission_reply(raw: str) -> str:
     line = (raw or "").strip()
     line = re.sub(r"^\s*[\-\*\u2022]\s*", "", line)
@@ -124,7 +155,7 @@ def _friendly_permission_reply(raw: str) -> str:
         return f"It seems to be an access issue: {line}. Please check your role mapping or request access."
     return line
 
-# ---------------- Language detection (Tamil/Hindi mirror) ----------------
+# ------------------ Language detection (Tamil/Hindi hint) ------------------
 def _detect_language_hint(msg: str) -> Optional[str]:
     if re.search(r"[\u0B80-\u0BFF]", msg or ""):  # Tamil
         return "Tamil"
@@ -132,14 +163,14 @@ def _detect_language_hint(msg: str) -> Optional[str]:
         return "Hindi"
     return None
 
-# ---------------- Clarifying message ----------------
+# ------------------ Clarifying message ------------------
 def _build_clarifying_message() -> str:
     return (
         "It seems the issue isn’t resolved yet. Would you like to share a few details so I can check further, "
         "or should I raise a ServiceNow ticket for you?"
     )
 
-# ---------------- Resolution/Incident helpers ----------------
+# ------------------ Resolution/Incident helpers ------------------
 def _build_tracking_descriptions(issue_text: str, resolved_text: str) -> Tuple[str, str]:
     issue = (issue_text or "").strip()
     resolved = (resolved_text or "").strip()
@@ -151,6 +182,7 @@ def _build_tracking_descriptions(issue_text: str, resolved_text: str) -> Tuple[s
     ).strip()
     return short_desc, long_desc
 
+
 def _is_incident_intent(msg_norm: str) -> bool:
     intent_phrases = [
         "create ticket", "create a ticket", "raise ticket", "raise a ticket", "open ticket", "open a ticket",
@@ -160,6 +192,7 @@ def _is_incident_intent(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in intent_phrases)
 
+
 def _parse_ticket_status_intent(msg_norm: str) -> Dict[str, Optional[str]]:
     status_keywords = ["status", "ticket status", "incident status", "check status", "check ticket status", "check incident status"]
     if not any(k in msg_norm for k in status_keywords):
@@ -173,6 +206,7 @@ def _parse_ticket_status_intent(msg_norm: str) -> Dict[str, Optional[str]]:
                 return {"number": val.upper() if val.lower().startswith("inc") else val}
     return {"number": None, "ask_number": True}
 
+
 def _is_resolution_ack_heuristic(msg_norm: str) -> bool:
     phrases = [
         "it is resolved", "resolved", "issue resolved", "problem resolved",
@@ -181,6 +215,7 @@ def _is_resolution_ack_heuristic(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in phrases)
 
+
 def _has_negation_resolved(msg_norm: str) -> bool:
     neg_phrases = [
         "not resolved", "issue not resolved", "still not working", "not working",
@@ -188,13 +223,12 @@ def _has_negation_resolved(msg_norm: str) -> bool:
     ]
     return any(p in msg_norm for p in neg_phrases)
 
+
 def _classify_resolution_llm(user_message: str) -> bool:
     if not GEMINI_API_KEY:
         return False
     prompt = f"""Classify if the following user message indicates that the issue is resolved or working now.
-
 Return only 'true' or 'false'.
-
 Message: {user_message}"""
     headers = {"Content-Type": "application/json"}
     payload = {"contents": [{"parts": [{"text": prompt}]}]}
@@ -211,26 +245,37 @@ Message: {user_message}"""
     except Exception:
         return False
 
-# ---------------- Filters ----------------
+# ------------------ Filters ------------------
 STRICT_OVERLAP = 3
 MAX_SENTENCES_STRICT = 4
 MAX_SENTENCES_CONCISE = 3
-
 PERM_QUERY_TERMS = [
     "permission", "permissions", "access", "access right", "authorization", "authorisation",
-    "role", "role access", "security", "security profile", "privilege", "not allowed", "not authorized", "denied"
+    "role", "role access", "security", "security profile", "privilege", "not allowed", "not authorized", "denied",
 ]
 
+# Permission-synonyms used to pick only permission-specific lines
+PERM_SYNONYMS = (
+    "permission", "permissions", "access", "authorization", "authorisation",
+    "role", "role mapping", "security profile", "not allowed", "not authorized", "denied", "insufficient"
+)
+
+
 def _normalize_for_match(text: str) -> str:
     t = (text or "").lower()
     t = re.sub(r"[^\w\s]", " ", t)
     t = re.sub(r"\s+", " ", t).strip()
     return t
 
+
 def _split_sentences(ctx: str) -> List[str]:
-    raw_sents = re.split(r"(?<=[.!?])\s+|\n+|\-\s*|\*\s*", ctx or "")
+    raw_sents = re.split(
+        r"(?<=([.!?]))\s+|\n+|\-\s*|\*\s*",
+        ctx or ""
+    )
     return [s.strip() for s in raw_sents if s and len(s.strip()) > 2]
 
+
 def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str, Any]]:
     ctx = (context or "").strip()
     if not ctx or not query:
@@ -239,6 +284,7 @@ def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str,
     q_terms = [t for t in q_norm.split() if len(t) > 2]
     if not q_terms:
         return ctx, {'mode': 'concise', 'matched_count': 0, 'all_sentences': 0}
+
     sentences = _split_sentences(ctx)
     matched_exact, matched_any = [], []
     for s in sentences:
@@ -249,6 +295,7 @@ def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str,
             matched_exact.append(s)
         elif overlap > 0:
             matched_any.append(s)
+
     if matched_exact:
         kept = matched_exact[:MAX_SENTENCES_STRICT]
         return "\n".join(kept).strip(), {'mode': 'exact', 'matched_count': len(kept), 'all_sentences': len(sentences)}
@@ -258,11 +305,12 @@ def _filter_context_for_query(context: str, query: str) -> Tuple[str, Dict[str,
     kept = sentences[:MAX_SENTENCES_CONCISE]
     return "\n".join(kept).strip(), {'mode': 'concise', 'matched_count': 0, 'all_sentences': len(sentences)}
 
-# ---------------- Navigation extraction ----------------
+# ------------------ Navigation & Errors extraction ------------------
 NAV_LINE_REGEX = re.compile(r"(navigate\s+to|login|log in|menu|screen)", re.IGNORECASE)
 
+
 def _extract_navigation_only(text: str, max_lines: int = 6) -> str:
-    lines = [ln.strip() for ln in (text or "").splitlines() if ln.strip()]
+    lines = _normalize_lines(text)
     kept: List[str] = []
     for ln in lines:
         if NAV_LINE_REGEX.search(ln) or ln.lower().startswith("log in") or ln.lower().startswith("login"):
@@ -271,15 +319,16 @@ def _extract_navigation_only(text: str, max_lines: int = 6) -> str:
             break
     return "\n".join(kept).strip() if kept else (text or "").strip()
 
-# ---------------- Errors extraction ----------------
+
 ERROR_STARTS = (
     "error", "resolution", "fix", "verify", "check",
     "permission", "access", "authorization", "authorisation",
     "role", "role mapping", "security profile", "escalation", "not allowed", "not authorized", "denied"
 )
 
+
 def _extract_errors_only(text: str, max_lines: int = 12) -> str:
-    lines = [ln.strip() for ln in (text or "").splitlines() if ln.strip()]
+    lines = _normalize_lines(text)
     kept: List[str] = []
     for ln in lines:
         low = ln.lower()
@@ -289,10 +338,27 @@ def _extract_errors_only(text: str, max_lines: int = 12) -> str:
             break
     return "\n".join(kept).strip() if kept else (text or "").strip()
 
+
+def _filter_permission_lines(text: str, max_lines: int = 6) -> str:
+    """
+    Keep only permission/access-related lines so the bot answers the exact thing user asked.
+    """
+    lines = _normalize_lines(text)
+    kept: List[str] = []
+    for ln in lines:
+        low = ln.lower()
+        if any(k in low for k in PERM_SYNONYMS):
+            kept.append(ln)
+        if len(kept) >= max_lines:
+            break
+    return "\n".join(kept).strip() if kept else (text or "").strip()
+
+# ------------------ Health ------------------
 @app.get("/")
 async def health_check():
     return {"status": "ok"}
 
+# ------------------ Chat ------------------
 @app.post("/chat")
 async def chat_with_ai(input_data: ChatInput):
     try:
@@ -477,20 +543,22 @@ async def chat_with_ai(input_data: ChatInput):
             comb = combined[i] if i < len(combined) else None
             m = dict(meta)
             if score is not None: m["distance"] = score
-            if comb is not None:  m["combined"] = comb
+            if comb is not None: m["combined"] = comb
             items.append({"text": text, "meta": m})
+
         selected = items[:max(1, 2)]
         context_raw = "\n\n---\n\n".join([s["text"] for s in selected]) if selected else ""
         filtered_text, filt_info = _filter_context_for_query(context_raw, input_data.user_message)
         context = filtered_text
         context_found = bool(context.strip())
+
         best_distance = min([d for d in distances if d is not None], default=None) if distances else None
         best_combined = max([c for c in combined if c is not None], default=None) if combined else None
         detected_intent = kb_results.get("user_intent", "neutral")
         best_doc = kb_results.get("best_doc")
         top_meta = (metadatas or [{}])[0] if metadatas else {}
 
-        # Force errors intent for permissions
+        # Force errors intent for permissions (and demote steps entirely)
         is_perm_query = any(t in msg_norm for t in PERM_QUERY_TERMS)
         if is_perm_query:
             detected_intent = "errors"
@@ -507,101 +575,101 @@ async def chat_with_ai(input_data: ChatInput):
             elif detected_intent == "errors":
                 full_errors = get_best_errors_section_text(best_doc)
                 if full_errors:
-                    context = _extract_errors_only(full_errors, max_lines=12)
-                    lines = _normalize_lines(context)
-                    if len(lines) == 1:
-                        context = _friendly_permission_reply(lines[0])
-
-        # Intent shaping
-        q = (input_data.user_message or "").lower()
-        if detected_intent == "errors" or any(k in q for k in [
-            "error", "issue", "fail", "not working", "resolution", "fix",
-            "permission", "access", "authorization", "escalation", "role", "security profile",
-            "not allowed", "not authorized", "denied"
-        ]):
-            context = _extract_errors_only(context, max_lines=12)
-            lines = _normalize_lines(context)
-            if len(lines) == 1:
-                context = _friendly_permission_reply(lines[0])
-        elif any(k in q for k in ["navigate", "navigation", "menu", "screen"]):
-            context = _extract_navigation_only(context, max_lines=6)
-        elif detected_intent == "steps":
-            context = _ensure_numbering(context)
+                    # Extract only error lines, then narrow to permission-only if the query is permission-related
+                    ctx_err = _extract_errors_only(full_errors, max_lines=12)
+                    context = _filter_permission_lines(ctx_err, max_lines=6) if is_perm_query else ctx_err
+                lines = _normalize_lines(context)
+                if len(lines) == 1:
+                    context = _friendly_permission_reply(lines[0])
+
+        # If we STILL have no context for an errors/permission query, return clarifying + ticket option
+        if detected_intent == "errors" and not (context or "").strip():
+            return {
+                "bot_response": _build_clarifying_message(),
+                "status": "NO_KB_MATCH",
+                "context_found": False,
+                "ask_resolved": False,
+                "suggest_incident": True,
+                "followup": "Share more details or say 'create ticket'.",
+                "options": [{"type":"yesno","title":"Share details or raise a ticket?"}],
+                "top_hits": [],
+                "sources": [],
+                "debug": {"intent": "errors_no_context"},
+            }
 
         # Build LLM prompt (mirror language if possible)
         language_hint = _detect_language_hint(input_data.user_message)
         lang_line = f"Respond in {language_hint}." if language_hint else "Respond in a clear, polite tone."
+
+        # Safer: skip Gemini for steps to avoid altering procedure; use Gemini only for errors/permissions.
+        use_gemini = (detected_intent == "errors")
         enhanced_prompt = f"""You are a helpful support assistant. Rewrite the provided context ONLY into clear, user-friendly guidance.
 - Do not add any information that is not present in the context.
-- If the content describes a procedure or steps, format as a strictly numbered list using plain text: 1), 2), 3), ...
 - If the content is an error/access/permission note, paraphrase it into a helpful sentence users can understand.
 - {lang_line}
-
 ### Context
 {context}
-
 ### Question
 {input_data.user_message}
-
 ### Output
 Return ONLY the rewritten guidance."""
+
         headers = {"Content-Type": "application/json"}
         payload = {"contents": [{"parts": [{"text": enhanced_prompt}]}]}
-        try:
-            resp = requests.post(GEMINI_URL, headers=headers, json=payload, timeout=25, verify=GEMINI_SSL_VERIFY)
+
+        bot_text = ""
+        http_code = 0
+        if use_gemini:
             try:
-                result = resp.json()
+                resp = requests.post(GEMINI_URL, headers=headers, json=payload, timeout=25, verify=GEMINI_SSL_VERIFY)
+                http_code = getattr(resp, "status_code", 0)
+                try:
+                    result = resp.json()
+                except Exception:
+                    result = {}
+                try:
+                    bot_text = result.get("candidates", [{}])[0].get("content", {}).get("parts", [{}])[0].get("text", "")
+                except Exception:
+                    bot_text = ""
             except Exception:
-                result = {}
-        except Exception:
-            resp = type("RespStub", (), {"status_code": 0})()
-            result = {}
-        try:
-            bot_text = result.get("candidates", [{}])[0].get("content", {}).get("parts", [{}])[0].get("text", "")
-        except Exception:
-            bot_text = ""
-        http_code = getattr(resp, "status_code", 0)
+                bot_text, http_code = "", 0
 
-        # Fallback to local formatting
-        if not bot_text.strip() or http_code == 429:
-            if detected_intent == "steps":
-                bot_text = _ensure_numbering(context)
-            elif detected_intent == "errors":
+        # Deterministic local formatting
+        if detected_intent == "steps":
+            bot_text = _ensure_numbering(context)
+        elif detected_intent == "errors":
+            if not bot_text.strip() or http_code == 429:
                 lines = _normalize_lines(context)
                 bot_text = _friendly_permission_reply(lines[0] if lines else context)
-            else:
-                bot_text = context
-
-        # Force numbering for steps
-        if detected_intent == "steps":
-            bot_text = _ensure_numbering(bot_text)
+            # Append standard escalation line for errors/permission messages
+            bot_text = (bot_text or "").rstrip() + "\n\nIf you want to escalate the issue, reach out to Operator – Inventory Analyst."
+        else:
+            bot_text = context
 
         # Status compute
         short_query = len((input_data.user_message or "").split()) <= 4
         gate_combined_ok = 0.60 if short_query else 0.55
         status = "OK" if (best_combined is not None and best_combined >= gate_combined_ok) else "PARTIAL"
-
-        lower = bot_text.lower()
+        lower = (bot_text or "").lower()
         if ("partial" in lower) or ("may be partial" in lower) or ("closest" in lower) or ("may not fully" in lower):
             status = "PARTIAL"
 
         options = [{"type":"yesno","title":"Share details or raise a ticket?"}] if status == "PARTIAL" else []
-
         return {
             "bot_response": bot_text,
             "status": status,
             "context_found": True,
-            "ask_resolved": (status == "OK"),
+            "ask_resolved": (status == "OK" and detected_intent != "errors"),  # don't ask resolved for error explanations
             "suggest_incident": (status == "PARTIAL"),
             "followup": ("Is this helpful, or should I raise a ticket?" if status == "PARTIAL" else None),
             "options": options,
             "top_hits": [],
             "sources": [],
             "debug": {
-                "used_chunks": len(context.split("\n\n---\n\n")) if context else 0,
+                "used_chunks": len((context or "").split("\n\n---\n\n")) if context else 0,
                 "best_distance": best_distance,
                 "best_combined": best_combined,
-                "http_status": getattr(resp, "status_code", 0),
+                "http_status": http_code,
                 "filter_mode": filt_info.get("mode"),
                 "matched_count": filt_info.get("matched_count"),
                 "user_intent": detected_intent,
@@ -614,6 +682,7 @@ Return ONLY the rewritten guidance."""
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
+
 def _set_incident_resolved(sys_id: str) -> bool:
     try:
         token = get_valid_token()
@@ -627,12 +696,14 @@ def _set_incident_resolved(sys_id: str) -> bool:
             "Content-Type": "application/json",
         }
         url = f"{instance_url}/api/now/table/incident/{sys_id}"
+
         close_code_val = os.getenv("SERVICENOW_CLOSE_CODE", "Solution provided")
         close_notes_val = os.getenv("SERVICENOW_RESOLUTION_NOTES", "Issue resolved, user confirmed")
         caller_sysid = os.getenv("SERVICENOW_CALLER_SYSID")
         resolved_by_sysid = os.getenv("SERVICENOW_RESOLVED_BY_SYSID")
         assign_group = os.getenv("SERVICENOW_ASSIGNMENT_GROUP_SYSID")
         require_progress = os.getenv("SERVICENOW_REQUIRE_IN_PROGRESS_FIRST", "false").lower() in ("1", "true", "yes")
+
         if require_progress:
             try:
                 resp1 = requests.patch(url, headers=headers, json={"state": "2"}, verify=VERIFY_SSL, timeout=25)
@@ -690,6 +761,7 @@ def _set_incident_resolved(sys_id: str) -> bool:
         print(f"[SN PATCH resolve] exception={safe_str(e)}")
         return False
 
+# ------------------ Incident creation ------------------
 @app.post("/incident")
 async def raise_incident(input_data: IncidentInput):
     try:
@@ -714,6 +786,7 @@ async def raise_incident(input_data: IncidentInput):
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
+# ------------------ Ticket description generation ------------------
 @app.post("/generate_ticket_desc")
 async def generate_ticket_desc_ep(input_data: TicketDescInput):
     try:
@@ -721,8 +794,8 @@ async def generate_ticket_desc_ep(input_data: TicketDescInput):
             f"You are helping generate ServiceNow ticket descriptions based on the issue: {input_data.issue}.\n"
             "Please return the output strictly in JSON format with the following keys:\n"
             "{\n"
-            '  "ShortDescription": "A concise summary of the issue (max 100 characters)",\n'
-            '  "DetailedDescription": "A detailed explanation of the issue"\n'
+            ' "ShortDescription": "A concise summary of the issue (max 100 characters)",\n'
+            ' "DetailedDescription": "A detailed explanation of the issue"\n'
             "}\n"
             "Do not include any extra text, comments, or explanations outside the JSON."
         )
@@ -751,6 +824,7 @@ async def generate_ticket_desc_ep(input_data: TicketDescInput):
     except Exception as e:
         raise HTTPException(status_code=500, detail=safe_str(e))
 
+# ------------------ Incident status ------------------
 @app.post("/incident_status")
 async def incident_status(input_data: TicketStatusInput):
     try:
@@ -759,6 +833,7 @@ async def incident_status(input_data: TicketStatusInput):
         if not instance_url:
             raise HTTPException(status_code=500, detail="SERVICENOW_INSTANCE_URL missing")
         headers = {"Authorization": f"Bearer {token}", "Accept": "application/json"}
+
         if input_data.sys_id:
             url = f"{instance_url}/api/now/table/incident/{input_data.sys_id}"
             response = requests.get(url, headers=headers, verify=VERIFY_SSL, timeout=25)
@@ -772,10 +847,12 @@ async def incident_status(input_data: TicketStatusInput):
             result = (lst or [{}])[0] if response.status_code == 200 else {}
         else:
             raise HTTPException(status_code=400, detail="Provide IncidentID (number) or sys_id")
+
         state_code = builtins.str(result.get("state", "unknown"))
         state_label = STATE_MAP.get(state_code, state_code)
         short = result.get("short_description", "")
         number = result.get("number", input_data.number or "unknown")
+
         return {
             "bot_response": (
                 f"**Ticket:** {number} \n"