Update app.py

app.py

@@ -4,7 +4,7 @@ import os
 import re
 import time
 from functools import lru_cache
-from typing import List, Dict, Any, Tuple
+from typing import List, Dict, Any, Tuple, Optional
 
 import numpy as np
 import pandas as pd
@@ -24,6 +24,11 @@ MODEL_NAME = os.getenv("HADITH_MODEL_NAME", "intfloat/multilingual-e5-base")
 DEFAULT_TOP_K = 10
 MAX_TOP_K = 50
 
+# pull more from FAISS then rerank by evidence
+DEFAULT_RERANK_K = 35
+MAX_RERANK_K = 120
+MIN_RERANK_K = 20
+
 DEFAULT_HL_TOPN = 6        # 0 = disable highlighting (FAST)
 MAX_HL_TOPN = 25
 
@@ -69,7 +74,42 @@ def escape_html(s: str) -> str:
 
 
 # =========================
-# Segmenting
+# Lexical match helpers
+# =========================
+AR_STOPWORDS = {
+    "من","الى","إلى","عن","على","في","و","ثم","أو","او","كما","كان","كانت","يكون","تكون",
+    "هذا","هذه","ذلك","تلك","هناك","هنا","هو","هي","هم","هن","أنا","انت","أنت","نحن",
+    "ما","ماذا","هل","لماذا","لم","لن","لا","إن","أن","إنه","أنه","إلا","الا","حتى","قد",
+    "كل","أي","أيّ","اي","ايًّا","أيضا","أيضًا","مع","عند","بين","بعد","قبل","إذا","اذ","إذ",
+    "قال","وقالت","يقول","يقولون","رسول","الله","صلى","عليه","وسلم"
+}
+
+def ar_tokens(text_norm: str) -> List[str]:
+    if not text_norm:
+        return []
+    t = re.sub(r"[^\u0600-\u06FF0-9\s]", " ", text_norm)
+    t = re.sub(r"\s+", " ", t).strip()
+    toks = [x for x in t.split(" ") if x and x not in AR_STOPWORDS and len(x) >= 2]
+    seen = set()
+    out = []
+    for w in toks:
+        if w not in seen:
+            seen.add(w)
+            out.append(w)
+    return out
+
+def lexical_match_ratio(query_norm: str, doc_norm: str) -> Tuple[float, List[str]]:
+    q_toks = ar_tokens(query_norm)
+    if not q_toks:
+        return 0.0, []
+    doc = " " + (doc_norm or "") + " "
+    matched = [w for w in q_toks if f" {w} " in doc]
+    ratio = len(matched) / max(1, len(q_toks))
+    return float(ratio), matched
+
+
+# =========================
+# Segmenting + isnad detection
 # =========================
 def split_ar_segments(text: str, max_len: int) -> List[str]:
     if not text:
@@ -93,11 +133,24 @@ def split_ar_segments(text: str, max_len: int) -> List[str]:
     if buf:
         segs.append(buf)
 
-    # fallback chunking
     if len(segs) <= 1 and len(t) > max_len:
         segs = [t[i:i+max_len].strip() for i in range(0, len(t), max_len) if t[i:i+max_len].strip()]
     return segs
 
+ISNAD_START = re.compile(r"^\s*(حدثنا|أخبرنا|أنبأنا|سمعت|حدثني|أخبرني|قال|عن)\b")
+NAME_HEURISTIC = re.compile(r"(بن|ابن|أبو|أبي|بنت)\s+\S+")
+
+def is_isnad_segment(seg: str) -> bool:
+    if not seg:
+        return False
+    s = seg.strip()
+    if ISNAD_START.search(s):
+        hits = len(NAME_HEURISTIC.findall(s))
+        chain_markers = sum(s.count(x) for x in [" عن ", " قال ", " حدثنا ", " أخبرنا ", " سمعت "])
+        if hits >= 1 or chain_markers >= 2 or len(s) < 120:
+            return True
+    return False
+
 
 # =========================
 # Load model + index + meta (once)
@@ -123,113 +176,231 @@ if "arabic_clean" not in meta.columns:
 # =========================
 # Embedding helpers (cached)
 # =========================
-@lru_cache(maxsize=1024)
+@lru_cache(maxsize=2048)
 def cached_query_emb(query_norm: str) -> bytes:
-    """Cache query embedding (normalized, float32). Return as bytes for caching."""
     emb = model.encode(["query: " + query_norm], normalize_embeddings=True).astype("float32")[0]
     return emb.tobytes()
 
 def get_query_emb(query_norm: str) -> np.ndarray:
     return np.frombuffer(cached_query_emb(query_norm), dtype=np.float32)
 
+def compute_segment_sims(query_emb: np.ndarray, segments: List[str]) -> np.ndarray:
+    if not segments:
+        return np.array([], dtype=np.float32)
+    seg_emb = model.encode(
+        ["passage: " + s for s in segments],
+        normalize_embeddings=True
+    ).astype("float32")
+    return (seg_emb @ query_emb).astype(np.float32)
 
-def semantic_search_df(query: str, top_k: int) -> pd.DataFrame:
-    q = str(query or "").strip()
-    if not q:
-        return meta.iloc[0:0].copy()
-
-    top_k = max(1, min(int(top_k), MAX_TOP_K))
-    q_norm = normalize_ar(q)
 
-    q_emb = get_query_emb(q_norm).reshape(1, -1)
-    scores, idx = index.search(q_emb, top_k)
+# =========================
+# Core search: candidates -> rerank by best MATN segment
+# =========================
+def search_candidates_df(query_norm: str, rerank_k: int) -> pd.DataFrame:
+    q_emb = get_query_emb(query_norm).reshape(1, -1)
+    scores, idxs = index.search(q_emb, rerank_k)
 
-    res = meta.iloc[idx[0]].copy()
-    res["score"] = scores[0]
-    res = res.sort_values("score", ascending=False)
+    res = meta.iloc[idxs[0]].copy()
+    res["faiss_score"] = scores[0]
+    res["faiss_rank"] = np.arange(len(res), dtype=np.int32)
 
     res["arabic"] = res["arabic"].fillna("").astype(str)
     res = res[res["arabic"].str.strip() != ""]
     return res
 
+def rerank_rows(query_norm: str, cand: pd.DataFrame, seg_maxlen: int) -> pd.DataFrame:
+    if cand.empty:
+        out = cand.copy()
+        out["score"] = np.nan
+        out["best_seg"] = ""
+        out["lex_ratio"] = 0.0
+        out["lex_terms"] = ""
+        return out
 
-# =========================
-# Batch semantic highlight (FAST)
-# =========================
-def build_highlight_html_batch(
-    query_norm: str,
-    arabic_clean_list: List[str],
-    hl_topn: int,
-    seg_maxlen: int,
-) -> Tuple[List[str], Dict[str, Any]]:
-    """
-    Return list of HTML strings (one per hadith), highlighted by segment similarity.
-    Uses ONE encode() call for all segments across all hadith results (fast).
-    """
-    # If disabled:
-    if hl_topn <= 0:
-        return [escape_html(t) for t in arabic_clean_list], {"mode": "disabled"}
-
-    # Split into segments per hadith
-    per_segments: List[List[str]] = [split_ar_segments(t, seg_maxlen) for t in arabic_clean_list]
-
-    # Flatten segments
-    all_segments: List[str] = []
-    offsets: List[Tuple[int,int]] = []  # (start, end) in flattened array
-    cur = 0
-    for segs in per_segments:
-        start = cur
-        all_segments.extend(segs)
-        cur += len(segs)
-        offsets.append((start, cur))
-
-    # Edge cases
-    if len(all_segments) == 0:
-        return [escape_html(t) for t in arabic_clean_list], {"mode": "empty"}
-
-    # Encode query once + encode all segments once
     q_emb = get_query_emb(query_norm)  # (d,)
-    seg_emb = model.encode(
-        ["passage: " + s for s in all_segments],
-        normalize_embeddings=True
-    ).astype("float32")  # (N, d)
 
-    sims_all = (seg_emb @ q_emb).astype(np.float32)  # (N,)
+    # Build per-row clean text
+    arabic_clean_list: List[str] = []
+    for _, row in cand.iterrows():
+        ar = str(row.get("arabic", "") or "")
+        ar_clean = row.get("arabic_clean", "")
+        if ar_clean is None or (isinstance(ar_clean, float) and np.isnan(ar_clean)):
+            ar_clean = ""
+        ar_clean = str(ar_clean).strip()
+        if not ar_clean:
+            ar_clean = normalize_ar(ar)
+        arabic_clean_list.append(ar_clean)
 
-    # Build HTML per hadith
-    html_out: List[str] = []
-    for (start, end), segs in zip(offsets, per_segments):
-        if start == end or len(segs) == 0:
-            html_out.append("")
-            continue
+    # Prepare segments for rerank (batch over all segments)
+    per_segments: List[List[str]] = []
+    per_deemph: List[np.ndarray] = []
+    all_segments: List[str] = []
+    seg_map: List[Tuple[int, int]] = []  # (row_i, seg_i_local)
+
+    for i, txt in enumerate(arabic_clean_list):
+        segs = split_ar_segments(txt, seg_maxlen)
+        if not segs:
+            segs = [txt] if txt else []
+        per_segments.append(segs)
+
+        deemph_mask = np.array([1.0 if not is_isnad_segment(s) else 0.0 for s in segs], dtype=np.float32)
+        per_deemph.append(deemph_mask)
+
+        for j, s in enumerate(segs):
+            all_segments.append(s)
+            seg_map.append((i, j))
+
+    if not all_segments:
+        out = cand.copy()
+        out["score"] = out["faiss_score"].astype(float)
+        out["best_seg"] = ""
+        out["lex_ratio"] = 0.0
+        out["lex_terms"] = ""
+        return out
+
+    # sims for all segments once
+    sims_all = compute_segment_sims(q_emb, all_segments)
+
+    # best segment per row (downweight isnad)
+    n_rows = len(per_segments)
+    best_sim = np.full((n_rows,), -1.0, dtype=np.float32)
+    best_local = np.full((n_rows,), -1, dtype=np.int32)
+
+    for k, (ri, sj) in enumerate(seg_map):
+        sim = float(sims_all[k])
+        deemph = float(per_deemph[ri][sj])  # 1 matn, 0 isnad-ish
+        sim_adj = sim * (0.70 + 0.30 * deemph)  # isnad gets downweighted
+        if sim_adj > best_sim[ri]:
+            best_sim[ri] = sim_adj
+            best_local[ri] = sj
+
+    # lexical match
+    lex_ratios: List[float] = []
+    lex_terms: List[str] = []
+    for txt in arabic_clean_list:
+        r, matched = lexical_match_ratio(query_norm, txt)
+        lex_ratios.append(r)
+        lex_terms.append("، ".join(matched[:10]) if matched else "")
+
+    out = cand.copy()
+    out["score"] = best_sim.astype(float)
+    out["best_seg_idx"] = best_local.astype(int)
+
+    # compute best_seg text
+    best_segs = []
+    for i, segs in enumerate(per_segments):
+        j = int(best_local[i])
+        best_segs.append(segs[j] if (0 <= j < len(segs)) else (segs[0] if segs else ""))
+    out["best_seg"] = best_segs
+
+    out["lex_ratio"] = np.array(lex_ratios, dtype=np.float32)
+    out["lex_terms"] = lex_terms
+
+    # Sort by evidence score, then faiss score, then original rank
+    out = out.sort_values(["score", "faiss_score", "faiss_rank"], ascending=[False, False, True])
+    return out
 
-        sims = sims_all[start:end]
-        s_min = float(np.min(sims))
-        s_max = float(np.max(sims))
-        denom = (s_max - s_min) if (s_max - s_min) > 1e-6 else 1.0
-
-        order = np.argsort(-sims)
-        keep = set(order[:min(hl_topn, len(segs))])
-
-        parts: List[str] = []
-        for i, seg in enumerate(segs):
-            w = (float(sims[i]) - s_min) / denom
-            alpha = (0.18 + 0.62 * w) if i in keep else (0.06 + 0.20 * w)
-            alpha = max(0.05, min(alpha, 0.82))
-            border_alpha = max(0.10, min(alpha * 0.8, 0.65))
-
-            style = (
-                f"background: rgba(255, 230, 120, {alpha:.3f});"
-                f"border: 1px solid rgba(234, 179, 8, {border_alpha:.3f});"
-                "border-radius: 12px;"
-                "padding: 3px 8px;"
-                "margin: 0 4px 6px 0;"
-                "display: inline;"
-            )
-            parts.append(f'<span style="{style}">{escape_html(seg)}</span> ')
-        html_out.append("".join(parts).strip())
 
-    return html_out, {"mode": "batch", "segments_total": len(all_segments)}
+# =========================
+# UI helpers for html mode
+# =========================
+def confidence_badge(score1: float, score2: Optional[float]) -> Tuple[str, str]:
+    s1 = float(score1) if score1 is not None else 0.0
+    s2 = float(score2) if score2 is not None else None
+    margin = (s1 - s2) if s2 is not None else s1
+
+    if s1 >= 0.78 or margin >= 0.08:
+        return "High", "bHigh"
+    if s1 >= 0.68 or margin >= 0.04:
+        return "Medium", "bMed"
+    return "Low", "bLow"
+
+def heatmap_html(sims: np.ndarray, bins: int = 16) -> str:
+    if sims.size == 0:
+        return ""
+    s_min = float(np.min(sims))
+    s_max = float(np.max(sims))
+    denom = (s_max - s_min) if (s_max - s_min) > 1e-6 else 1.0
+
+    n = sims.size
+    if n <= bins:
+        take_idx = list(range(n))
+    else:
+        take_idx = [int(round(i)) for i in np.linspace(0, n - 1, bins)]
+
+    parts = []
+    for i in take_idx:
+        w = (float(sims[i]) - s_min) / denom
+        alpha = 0.10 + 0.75 * w
+        alpha = max(0.08, min(alpha, 0.90))
+        parts.append(
+            f'<span title="seg {i+1}" style="display:inline-block;width:10px;height:10px;'
+            f'margin:0 2px;border-radius:3px;background:rgba(37,99,235,{alpha:.3f});"></span>'
+        )
+    return '<div style="margin:10px 0 8px;direction:ltr;text-align:left;">' + "".join(parts) + "</div>"
+
+def highlight_segments_html(segs: List[str], sims: np.ndarray, strong_topn: int, deemph_mask: np.ndarray) -> str:
+    if not segs or sims.size == 0:
+        return ""
+
+    s_min = float(np.min(sims))
+    s_max = float(np.max(sims))
+    denom = (s_max - s_min) if (s_max - s_min) > 1e-6 else 1.0
+
+    order = np.argsort(-sims)
+    keep = set(order[:min(strong_topn, len(segs))])
+
+    parts: List[str] = []
+    for i, seg in enumerate(segs):
+        w = (float(sims[i]) - s_min) / denom
+
+        deemph = float(deemph_mask[i])  # 1 matn, 0 isnad-like
+        alpha = (0.18 + 0.62 * w) if i in keep else (0.06 + 0.20 * w)
+        alpha = alpha * (0.45 + 0.55 * deemph)
+        alpha = max(0.04, min(alpha, 0.82))
+        border_alpha = max(0.08, min(alpha * 0.75, 0.60))
+
+        style = (
+            f"background: rgba(255, 230, 120, {alpha:.3f});"
+            f"border: 1px solid rgba(234, 179, 8, {border_alpha:.3f});"
+            "border-radius: 12px;"
+            "padding: 3px 8px;"
+            "margin: 0 4px 6px 0;"
+            "display: inline;"
+        )
+        parts.append(f'<span style="{style}">{escape_html(seg)}</span> ')
+    return "".join(parts).strip()
+
+def build_html_extras_for_row(query_norm: str, arabic_clean_text: str, hl_topn: int, seg_maxlen: int) -> Dict[str, str]:
+    segs = split_ar_segments(arabic_clean_text, seg_maxlen)
+    if not segs:
+        segs = [arabic_clean_text] if arabic_clean_text else []
+
+    deemph_mask = np.array([1.0 if not is_isnad_segment(s) else 0.0 for s in segs], dtype=np.float32)
+    q_emb = get_query_emb(query_norm)
+    sims = compute_segment_sims(q_emb, segs)
+
+    hm = heatmap_html(sims, bins=16) if hl_topn > 0 else ""
+    highlighted = highlight_segments_html(segs, sims, strong_topn=max(1, hl_topn), deemph_mask=deemph_mask) if hl_topn > 0 else escape_html(arabic_clean_text)
+
+    # Best seg
+    best_seg = ""
+    if sims.size > 0:
+        best_i = int(np.argmax(sims))
+        best_seg = segs[best_i]
+
+    best_seg_html = (
+        f'<span style="background:rgba(255,230,120,.55);border:1px solid rgba(234,179,8,.40);'
+        f'border-radius:12px;padding:4px 10px;display:inline;">{escape_html(best_seg)}</span>'
+        if best_seg else ""
+    )
+
+    return {
+        "heatmap_html": hm,
+        "arabic_clean_html": highlighted if highlighted else escape_html(arabic_clean_text),
+        "best_seg_html": best_seg_html or "",
+    }
 
 
 # =========================
@@ -237,6 +408,7 @@ def build_highlight_html_batch(
 # =========================
 app = Flask(__name__)
 
+# (UI is optional for HF; keep it for quick testing)
 UI_HTML = r"""
 <!doctype html>
 <html lang="ar" dir="rtl">
@@ -251,6 +423,7 @@ UI_HTML = r"""
     :root{
       --bg:#f6f7fb; --card:#ffffff; --text:#0f172a; --muted:#475569;
       --line:#e5e7eb; --accent:#2563eb; --shadow: 0 10px 30px rgba(15, 23, 42, .08);
+      --good:#16a34a; --warn:#f59e0b; --bad:#ef4444;
     }
     body{
       margin:0; background: linear-gradient(180deg, #ffffff, var(--bg)); color: var(--text);
@@ -296,26 +469,45 @@ UI_HTML = r"""
       background: var(--card); border: 1px solid var(--line); border-radius:18px;
       padding: 16px; box-shadow: var(--shadow);
     }
-    .row{ display:grid; grid-template-columns: 210px 1fr; gap:14px; }
+    .row{ display:grid; grid-template-columns: 240px 1fr; gap:14px; }
     @media (max-width: 900px){ .row{ grid-template-columns: 1fr; } }
 
     .left{ color: var(--muted); font-size:14px; direction:ltr; text-align:left; }
-    .score{ font-weight:800; color: var(--accent); font-size:16px; }
+    .score{ font-weight:900; color: var(--accent); font-size:18px; }
+    .badge{
+      display:inline-flex; align-items:center; gap:6px; border-radius:999px;
+      padding:5px 10px; font-weight:900; font-size:12px; margin-top:10px;
+      border:1px solid var(--line); background:#fff;
+    }
+    .bHigh{ color: var(--good); border-color: rgba(22,163,74,.35); background: rgba(22,163,74,.08); }
+    .bMed{ color: var(--warn); border-color: rgba(245,158,11,.35); background: rgba(245,158,11,.10); }
+    .bLow{ color: var(--bad); border-color: rgba(239,68,68,.35); background: rgba(239,68,68,.08); }
 
     .arabic{
       direction: rtl; text-align:right; font-family: Amiri, serif; font-size:22px;
       line-height: 2.05; background:#fbfcff; border:1px solid var(--line);
       border-radius:16px; padding:14px; white-space: pre-wrap;
     }
+    .evidence{
+      margin-top: 10px; border: 1px dashed rgba(37,99,235,.25);
+      background: rgba(37,99,235,.05); border-radius: 14px;
+      padding: 10px 12px; direction: rtl; text-align: right;
+      font-family: Amiri, serif; font-size: 18px; line-height: 1.95;
+    }
+    .evidence small{
+      display:block; margin-bottom:6px; font-family: Tajawal, sans-serif;
+      color: var(--muted); direction:ltr; text-align:left; font-size:12px;
+    }
     .english{
       direction:ltr; text-align:left; font-size:16px; line-height:1.8; color:#111827;
       background:#fbfcff; border:1px solid var(--line); border-radius:16px; padding:14px; white-space: pre-wrap;
     }
     details summary{
       cursor:pointer; color: var(--accent); margin-top:12px; user-select:none;
-      direction:ltr; text-align:left; font-weight:700;
+      direction:ltr; text-align:left; font-weight:800;
     }
     .empty{ margin-top: 14px; color: var(--muted); font-size: 15px; direction:ltr; text-align:left; }
+    .tiny{ margin-top:8px; font-size:12px; color: var(--muted); direction:ltr; text-align:left; }
   </style>
 </head>
 <body>
@@ -332,15 +524,14 @@ UI_HTML = r"""
       </form>
 
       <div class="controls">
-        <label>
-          Highlight Top Segments:
-          <input id="hl" type="range" min="0" max="25" value="6">
-          <b id="hlv">6</b>
+        <label>Highlight Top Segments:
+          <input id="hl" type="range" min="0" max="25" value="6"><b id="hlv">6</b>
+        </label>
+        <label>Segment Size:
+          <input id="seg" type="range" min="120" max="420" step="20" value="220"><b id="segv">220</b>
         </label>
-        <label>
-          Segment Size:
-          <input id="seg" type="range" min="120" max="420" step="20" value="220">
-          <b id="segv">220</b>
+        <label>Re-rank pool:
+          <input id="rk" type="range" min="20" max="120" step="5" value="35"><b id="rkv">35</b>
         </label>
       </div>
 
@@ -365,7 +556,7 @@ UI_HTML = r"""
     l.textContent = r.value;
     r.addEventListener("input", ()=> l.textContent = r.value);
   }
-  sync("hl","hlv"); sync("seg","segv");
+  sync("hl","hlv"); sync("seg","segv"); sync("rk","rkv");
 
   $("f").addEventListener("submit", async (e)=>{
     e.preventDefault();
@@ -373,11 +564,11 @@ UI_HTML = r"""
     const k = parseInt($("k").value||"10",10);
     const hl = parseInt($("hl").value||"6",10);
     const seg = parseInt($("seg").value||"220",10);
+    const rk  = parseInt($("rk").value||"35",10);
 
     $("msg").style.display="none";
     $("grid").innerHTML = "";
     $("meta").style.display="none";
-    $("meta").innerHTML = pill("Query", q) + pill("TopK", k) + pill("Highlight", hl) + pill("SegLen", seg);
 
     if(!q){
       $("msg").textContent="اكتب نص البحث أولًا.";
@@ -388,15 +579,14 @@ UI_HTML = r"""
     $("msg").textContent="... جاري البحث";
     $("msg").style.display="block";
 
-    const url = `/search?q=${encodeURIComponent(q)}&k=${encodeURIComponent(k)}&hl_topn=${encodeURIComponent(hl)}&seg_maxlen=${encodeURIComponent(seg)}&format=html`;
-    const t0 = performance.now();
+    const url = `/search?q=${encodeURIComponent(q)}&k=${encodeURIComponent(k)}&rerank_k=${encodeURIComponent(rk)}&hl_topn=${encodeURIComponent(hl)}&seg_maxlen=${encodeURIComponent(seg)}&format=html`;
     const res = await fetch(url);
     const js = await res.json();
-    const ms = Math.round(performance.now()-t0);
 
     $("meta").style.display="flex";
     $("meta").innerHTML =
-      pill("Rows", js.rows) + pill("Results", js.n) + pill("Time", js.took_ms ?? ms) + pill("TopK", js.k) + pill("Query", js.query);
+      pill("Rows", js.rows) + pill("Results", js.n) + pill("Time(ms)", js.took_ms) +
+      pill("TopK", js.k) + pill("ReRank", js.rerank_k) + pill("Query", js.query);
 
     if(!js.ok || !js.results || js.results.length===0){
       $("msg").textContent="لا توجد نتائج. جرّب كلمات مختلفة.";
@@ -406,25 +596,42 @@ UI_HTML = r"""
     $("msg").style.display="none";
 
     const cards = js.results.map(r=>{
-      const ar = js.format==="html" ? (r.arabic_clean_html||esc(r.arabic_clean||"")) : esc(r.arabic_clean||"");
+      const hm = r.heatmap_html || "";
+      const best = r.best_seg_html || "";
+      const ar = r.arabic_clean_html || esc(r.arabic_clean||"");
       const ar_tashkeel = esc(r.arabic||"");
       const en = esc(r.english||"");
+
       return `
         <div class="card">
           <div class="row">
             <div class="left">
-              <div><span class="score">${Number(r.score||0).toFixed(4)}</span> score</div>
+              <div><span class="score">${Number(r.score||0).toFixed(4)}</span> evidence</div>
+              <div class="tiny">FAISS: <b>${Number(r.faiss_score||0).toFixed(4)}</b></div>
+
+              <div class="badge ${esc(r.conf_class||"")}">Confidence: <b>${esc(r.conf_label||"")}</b></div>
+
+              <div class="tiny" style="margin-top:10px;">
+                Lexical match: <b>${Math.round((r.lex_ratio||0)*100)}%</b>
+                ${r.lex_terms ? `<div style="margin-top:6px;">Matched: <b>${esc(r.lex_terms)}</b></div>` : ``}
+              </div>
+
               <div style="margin-top:12px;">HadithID: <b>${esc(r.hadithID)}</b></div>
               <div>Collection: <b>${esc(r.collection)}</b></div>
               <div>No: <b>${esc(r.hadith_number)}</b></div>
             </div>
+
             <div>
-              <div class="arabic">${ar}</div>
+              ${hm}
+              <div class="evidence"><small>Top evidence snippet</small>${best}</div>
+              <div class="arabic" style="margin-top:10px;">${ar}</div>
+
               <details>
                 <summary>Show Arabic with tashkeel</summary>
                 <div style="height:10px;"></div>
                 <div class="arabic">${ar_tashkeel}</div>
               </details>
+
               <details>
                 <summary>Show English</summary>
                 <div style="height:10px;"></div>
@@ -476,6 +683,15 @@ def search():
         k = DEFAULT_TOP_K
     k = max(1, min(k, MAX_TOP_K))
 
+    # rerank pool
+    rk_raw = request.args.get("rerank_k", str(DEFAULT_RERANK_K)).strip()
+    try:
+        rerank_k = int(rk_raw) if rk_raw else DEFAULT_RERANK_K
+    except Exception:
+        rerank_k = DEFAULT_RERANK_K
+    rerank_k = max(MIN_RERANK_K, min(rerank_k, MAX_RERANK_K))
+    rerank_k = max(rerank_k, k)  # must be >= k
+
     # Highlight controls
     hl_raw = request.args.get("hl_topn", str(DEFAULT_HL_TOPN)).strip()
     seg_raw = request.args.get("seg_maxlen", str(DEFAULT_SEG_MAXLEN)).strip()
@@ -500,6 +716,7 @@ def search():
             "query": "",
             "query_norm": "",
             "k": k,
+            "rerank_k": rerank_k,
             "n": 0,
             "rows": int(len(meta)),
             "took_ms": 0,
@@ -508,49 +725,73 @@ def search():
         })
 
     t0 = time.time()
-    df = semantic_search_df(q, top_k=k)
+    q_norm = normalize_ar(q)
+
+    # 1) candidates from FAISS
+    cand = search_candidates_df(q_norm, rerank_k=rerank_k)
+
+    # 2) rerank by MATN evidence
+    reranked = rerank_rows(q_norm, cand, seg_maxlen=seg_maxlen)
+
+    # 3) take top k
+    reranked = reranked.head(k).copy()
     took_ms = int((time.time() - t0) * 1000)
 
-    q_norm = normalize_ar(q)
+    # confidence uses margin between first and second
+    scores_final = reranked["score"].astype(float).tolist()
+    top2 = scores_final[1] if len(scores_final) > 1 else None
+
+    results: List[Dict[str, Any]] = []
+    for pos, (_, row) in enumerate(reranked.iterrows()):
+        arabic = str(row.get("arabic", "") or "")
+        english = str(row.get("english", "") or "")
 
-    # Build clean arabic list (fallback derive if missing)
-    arabic_list: List[str] = []
-    for _, row in df.iterrows():
-        ar = str(row.get("arabic", "") or "")
         ar_clean = row.get("arabic_clean", "")
         if ar_clean is None or (isinstance(ar_clean, float) and np.isnan(ar_clean)):
             ar_clean = ""
         ar_clean = str(ar_clean).strip()
         if not ar_clean:
-            ar_clean = normalize_ar(ar)
-        arabic_list.append(ar_clean)
-
-    # Highlight (batch)
-    ar_html_list: List[str] = ["" for _ in arabic_list]
-    dbg: Dict[str, Any] = {}
-    if want_html:
-        ar_html_list, dbg = build_highlight_html_batch(
-            query_norm=q_norm,
-            arabic_clean_list=arabic_list,
-            hl_topn=hl_topn,
-            seg_maxlen=seg_maxlen,
-        )
+            ar_clean = normalize_ar(arabic)
 
-    results: List[Dict[str, Any]] = []
-    for i, (_, row) in enumerate(df.iterrows()):
-        arabic = str(row.get("arabic", "") or "")
-        english = str(row.get("english", "") or "")
-        r = {
+        score = float(row.get("score")) if pd.notna(row.get("score")) else 0.0
+
+        # confidence
+        if pos == 0:
+            conf_label, conf_class = confidence_badge(score, top2)
+        else:
+            # compare against top1 as rough indicator
+            conf_label, conf_class = confidence_badge(score, scores_final[0] if scores_final else None)
+
+        r: Dict[str, Any] = {
             "hadithID": int(row.get("hadithID")) if pd.notna(row.get("hadithID")) else None,
             "collection": str(row.get("collection", "") or ""),
             "hadith_number": int(row.get("hadith_number")) if pd.notna(row.get("hadith_number")) else None,
-            "score": float(row.get("score")) if pd.notna(row.get("score")) else 0.0,
+
+            "score": score,                # evidence score (reranked)
+            "faiss_score": float(row.get("faiss_score") or 0.0),
+            "faiss_rank": int(row.get("faiss_rank") or 0),
+
+            "lex_ratio": float(row.get("lex_ratio") or 0.0),
+            "lex_terms": str(row.get("lex_terms", "") or ""),
+
+            "conf_label": conf_label,
+            "conf_class": conf_class,
+
             "arabic": arabic,
-            "arabic_clean": arabic_list[i],
+            "arabic_clean": ar_clean,
             "english": english,
+
+            "best_seg": str(row.get("best_seg", "") or ""),
         }
+
         if want_html:
-            r["arabic_clean_html"] = ar_html_list[i] if ar_html_list[i] else escape_html(arabic_list[i])
+            extras = build_html_extras_for_row(
+                query_norm=q_norm,
+                arabic_clean_text=ar_clean,
+                hl_topn=hl_topn,
+                seg_maxlen=seg_maxlen,
+            )
+            r.update(extras)
         results.append(r)
 
     return jsonify({
@@ -558,17 +799,16 @@ def search():
         "query": q,
         "query_norm": q_norm,
         "k": k,
+        "rerank_k": rerank_k,
         "n": len(results),
         "rows": int(len(meta)),
         "took_ms": took_ms,
         "format": "html" if want_html else "json",
         "hl_topn": hl_topn,
         "seg_maxlen": seg_maxlen,
-        "debug": dbg if want_html else {},
         "results": results,
     })
 
 
 if __name__ == "__main__":
-    # local run only
     app.run(host="127.0.0.1", port=5000, debug=True)