Update rag_treatment_app.py

rag_treatment_app.py

@@ -103,9 +103,17 @@ class RAGTreatmentSearchApp:
         self.web_max_docs = int(os.getenv("WEB_MAX_DOCS", "6"))
         self.web_max_chars = int(os.getenv("WEB_MAX_CHARS", "1200"))
 
-        # NEW: hard gate to prevent "empty issue" generic recommendations
+        # NEW: hard gate to prevent empty/generic recommendation runs
         self.min_issue_chars = int(os.getenv("MIN_ISSUE_CHARS", "5"))
 
+        # NEW: mismatch sensitivity knobs
+        # If issue-only similarity within selected region is below this => likely irrelevant
+        self.local_issue_min_sim = float(os.getenv("LOCAL_ISSUE_MIN_SIM", "0.42"))
+        # If global best similarity is above this => the issue clearly maps somewhere else
+        self.global_issue_min_sim = float(os.getenv("GLOBAL_ISSUE_MIN_SIM", "0.52"))
+        # Additional delta guard (keeps your old behavior but improved)
+        self.global_local_delta = float(os.getenv("GLOBAL_LOCAL_DELTA", "0.10"))
+
     # ---------------- DB ----------------
     def _load_db(self) -> pd.DataFrame:
         xl = pd.ExcelFile(self.excel_path)
@@ -250,6 +258,42 @@ class RAGTreatmentSearchApp:
                 break
         return out
 
+    def _local_issue_only_best_sim(self, region: str, sub_zone: str, type_choice: str, issue_text: str) -> float:
+        """
+        NEW: Compute issue-only similarity *within selected region/subzone*.
+        This avoids the bias from including region/subzone words in the query.
+        """
+        issue_text = (issue_text or "").strip()
+        if not issue_text:
+            return 0.0
+
+        # consider both types if user chose both
+        t = _norm_type_choice(type_choice)
+        if t == "both":
+            idx_s = self._candidate_indices(region, sub_zone, "surgical")
+            idx_n = self._candidate_indices(region, sub_zone, "non-surgical")
+            idxs = np.unique(np.concatenate([idx_s, idx_n])) if (idx_s.size or idx_n.size) else np.array([], dtype=int)
+        else:
+            idxs = self._candidate_indices(region, sub_zone, t)
+
+        if idxs.size == 0:
+            # fall back to region only, still issue-only
+            if t == "both":
+                idx_s = self._candidate_indices(region, "", "surgical")
+                idx_n = self._candidate_indices(region, "", "non-surgical")
+                idxs = np.unique(np.concatenate([idx_s, idx_n])) if (idx_s.size or idx_n.size) else np.array([], dtype=int)
+            else:
+                idxs = self._candidate_indices(region, "", t)
+
+        if idxs.size == 0:
+            return 0.0
+
+        q_emb = self.model.encode([issue_text], convert_to_numpy=True).astype(np.float32)
+        sims = cosine_similarity(q_emb, self.embeddings[idxs])[0]
+        if sims.size == 0:
+            return 0.0
+        return float(np.max(sims))
+
     def semantic_search(self, region: str, sub_zone: str, type_choice: str, issue_text: str, top_k: int = 12) -> List[RetrievedCandidate]:
         type_norm = _norm_type_choice(type_choice)
         query = f"Region: {region} | Sub-Zone: {sub_zone} | Preference: {type_choice} | Issue: {issue_text}"
@@ -303,8 +347,12 @@ class RAGTreatmentSearchApp:
         return docs
 
     def _web_enrich_fallback(self, procedure: str) -> List[WebDoc]:
+        """
+        Second-pass retrieval only if extraction is failing.
+        """
         if not self.web_enabled:
             return []
+
         queries = [
             f"{procedure} recovery swelling bruising downtime",
             f"{procedure} procedure time how long does it take",
@@ -385,12 +433,12 @@ Extract patient-facing procedure details for: {procedure}
 Use ONLY the evidence below. If not present, write "Not found in evidence."
 
 Return STRICT JSON with these keys:
-- invasiveness
-- duration
-- downtime
-- longevity
-- risks
-- best_suited_for
+- invasiveness                (Non-invasive / Minimally invasive / Surgical, or evidence-based wording)
+- duration                    (typical treatment/procedure time; include units if present)
+- downtime                    (recovery/downtime; typical range)
+- longevity                   (how long results last; typical range)
+- risks                       (common risks/side effects; concise)
+- best_suited_for             (who it is for; concise)
 
 Evidence:
 {evidence}
@@ -471,7 +519,7 @@ Evidence:
         retrieval_k: int = 12,
         final_k: int = 5,
     ) -> Dict[str, object]:
-        # -------- NEW: Hard input gating to prevent empty issue searches --------
+        # --- Hard gate (must have region, sub-zone, and meaningful issue text) ---
         region = (region or "").strip()
         sub_zone = (sub_zone or "").strip()
         issue_text = (issue_text or "").strip()
@@ -493,7 +541,7 @@ Evidence:
 
         candidates = self.semantic_search(region, sub_zone, type_choice, issue_text, top_k=int(retrieval_k))
 
-        # If no candidates after filtering, return friendly message
+        # If nothing returned locally, show friendly message
         if not candidates:
             return {
                 "answer_md": "No matching procedures found for your selected Region/Sub-Zone and issue. Please revise your inputs.",
@@ -501,13 +549,14 @@ Evidence:
                 "_debug": {"mismatch": False, "candidate_count": 0, "final_count": 0},
             }
 
-        # Mismatch check using global signal (kept conservative)
+        # ---------------- Improved mismatch detection ----------------
         global_cands = self._global_semantic(issue_text, top_k=15)
         global_best = global_cands[0].similarity if global_cands else 0.0
         local_best = candidates[0].similarity if candidates else 0.0
 
         selected_region_norm = _norm(region)
         selected_sub_norm = _norm(sub_zone)
+
         selected_in_global = any(
             _norm(c.region) == selected_region_norm and (
                 selected_sub_norm in _norm(c.sub_zone) or _norm(c.sub_zone) in selected_sub_norm
@@ -515,7 +564,27 @@ Evidence:
             for c in global_cands[:10]
         )
 
-        if (global_best >= 0.50 and (global_best - local_best) >= 0.12 and not selected_in_global):
+        # NEW: issue-only best similarity inside selected region/subzone
+        local_issue_best = self._local_issue_only_best_sim(region, sub_zone, type_choice, issue_text)
+
+        # Trigger mismatch if:
+        # - issue-only relevance to selected region is low
+        # - but global mapping is strong
+        # - and global top does not align with selected region/subzone
+        mismatch_strict = (
+            (local_issue_best > 0.0 and local_issue_best < self.local_issue_min_sim) and
+            (global_best >= self.global_issue_min_sim) and
+            (not selected_in_global)
+        )
+
+        # Keep your older delta-based signal (still useful)
+        mismatch_delta = (
+            (global_best >= self.global_issue_min_sim) and
+            ((global_best - local_best) >= self.global_local_delta) and
+            (not selected_in_global)
+        )
+
+        if mismatch_strict or mismatch_delta:
             suggestions = []
             seen = set()
             for c in global_cands:
@@ -531,13 +600,24 @@ Evidence:
             answer_md = f"""## Notice
 Sorry for inconvenience. Your selected body region/sub-zone is not appropriate as per your defined problem.
 
-## Suggested Region/Sub-Zones
+## Suggested Region/Sub-Zones (based on your issue text)
 {sug_lines}
 
 ## Next step
 Please select one of the suggested Region/Sub-Zones and run the search again.
 """.strip()
-            return {"answer_md": answer_md, "sources": [], "_debug": {"mismatch": True}}
+
+            return {
+                "answer_md": answer_md,
+                "sources": [],
+                "_debug": {
+                    "mismatch": True,
+                    "global_best": round(global_best, 4),
+                    "local_best": round(local_best, 4),
+                    "local_issue_best": round(local_issue_best, 4),
+                },
+            }
+        # -------------------------------------------------------------
 
         best = self._llm_rerank(issue_text, candidates, top_k=int(final_k))
         if len(best) < int(final_k):
@@ -559,7 +639,6 @@ Please select one of the suggested Region/Sub-Zones and run the search again.
 
         answer_md = self._format_final_answer(best, web_by_proc)
 
-        # de-dupe urls
         seen_u = set()
         dedup = []
         for u in urls: