Update app.py

app.py

@@ -3,13 +3,14 @@ from __future__ import annotations
 import os
 import re
 import time
+from functools import lru_cache
 from typing import List, Dict, Any, Tuple
 
 import numpy as np
 import pandas as pd
 import faiss
-from flask import Flask, request, jsonify
-from flask_cors import CORS
+
+from flask import Flask, request, jsonify, Response
 from sentence_transformers import SentenceTransformer
 
 
@@ -23,11 +24,12 @@ MODEL_NAME = os.getenv("HADITH_MODEL_NAME", "intfloat/multilingual-e5-base")
 DEFAULT_TOP_K = 10
 MAX_TOP_K = 50
 
-DEFAULT_HL_TOPN = 6
+DEFAULT_HL_TOPN = 6        # 0 = disable highlighting (FAST)
 MAX_HL_TOPN = 25
 
-DEFAULT_SEG_MAXLEN = 220
+DEFAULT_SEG_MAXLEN = 220   # segment size
 MAX_SEG_MAXLEN = 420
+MIN_SEG_MAXLEN = 120
 
 
 # =========================
@@ -67,7 +69,7 @@ def escape_html(s: str) -> str:
 
 
 # =========================
-# Segmenting + semantic highlight
+# Segmenting
 # =========================
 def split_ar_segments(text: str, max_len: int) -> List[str]:
     if not text:
@@ -91,74 +93,17 @@ def split_ar_segments(text: str, max_len: int) -> List[str]:
     if buf:
         segs.append(buf)
 
+    # fallback chunking
     if len(segs) <= 1 and len(t) > max_len:
         segs = [t[i:i+max_len].strip() for i in range(0, len(t), max_len) if t[i:i+max_len].strip()]
-
     return segs
 
-def semantic_highlight_segments_html(
-    model: SentenceTransformer,
-    query_norm: str,
-    arabic_clean: str,
-    top_n: int,
-    seg_max_len: int
-) -> Tuple[str, List[Dict[str, Any]]]:
-    """
-    Returns:
-      - HTML string with highlighted segments
-      - segments_debug: list of {seg, sim, strong}
-    """
-    segs = split_ar_segments(arabic_clean, max_len=seg_max_len)
-    if not segs:
-        return escape_html(arabic_clean), []
-
-    q_emb = model.encode(["query: " + query_norm], normalize_embeddings=True).astype("float32")
-    seg_emb = model.encode(["passage: " + s for s in segs], normalize_embeddings=True).astype("float32")
-
-    sims = (seg_emb @ q_emb[0]).astype(np.float32)
-    s_min = float(np.min(sims))
-    s_max = float(np.max(sims))
-    denom = (s_max - s_min) if (s_max - s_min) > 1e-6 else 1.0
-
-    order = np.argsort(-sims)
-    keep = set(order[:min(top_n, len(segs))])
-
-    html_parts: List[str] = []
-    dbg: List[Dict[str, Any]] = []
-
-    for i, seg in enumerate(segs):
-        w = (float(sims[i]) - s_min) / denom  # 0..1
-        strong = i in keep
-
-        # Strong highlight for top segments, softer for others
-        alpha = (0.18 + 0.62 * w) if strong else (0.06 + 0.20 * w)
-        alpha = max(0.05, min(alpha, 0.82))
-        border_alpha = max(0.10, min(alpha * 0.8, 0.65))
-
-        style = (
-            f"background: rgba(255, 230, 120, {alpha:.3f});"
-            f"border: 1px solid rgba(234, 179, 8, {border_alpha:.3f});"
-            "border-radius: 12px;"
-            "padding: 3px 8px;"
-            "margin: 0 4px 6px 0;"
-            "display: inline;"
-        )
-        html_parts.append(f'<span style="{style}">{escape_html(seg)}</span> ')
-        dbg.append({"seg": seg, "sim": float(sims[i]), "strong": bool(strong)})
-
-    html = "".join(html_parts).strip()
-    if not html:
-        html = escape_html(arabic_clean)
-
-    return html, dbg
-
 
 # =========================
 # Load model + index + meta (once)
 # =========================
 if not os.path.exists(INDEX_PATH):
     raise FileNotFoundError(f"FAISS index not found: {INDEX_PATH}")
-
 if not os.path.exists(META_PATH):
     raise FileNotFoundError(f"Meta parquet not found: {META_PATH}")
 
@@ -175,16 +120,28 @@ if "arabic_clean" not in meta.columns:
     meta["arabic_clean"] = ""
 
 
-def semantic_search(query: str, top_k: int) -> pd.DataFrame:
+# =========================
+# Embedding helpers (cached)
+# =========================
+@lru_cache(maxsize=1024)
+def cached_query_emb(query_norm: str) -> bytes:
+    """Cache query embedding (normalized, float32). Return as bytes for caching."""
+    emb = model.encode(["query: " + query_norm], normalize_embeddings=True).astype("float32")[0]
+    return emb.tobytes()
+
+def get_query_emb(query_norm: str) -> np.ndarray:
+    return np.frombuffer(cached_query_emb(query_norm), dtype=np.float32)
+
+
+def semantic_search_df(query: str, top_k: int) -> pd.DataFrame:
     q = str(query or "").strip()
     if not q:
         return meta.iloc[0:0].copy()
 
     top_k = max(1, min(int(top_k), MAX_TOP_K))
-
     q_norm = normalize_ar(q)
-    q_emb = model.encode(["query: " + q_norm], normalize_embeddings=True).astype("float32")
 
+    q_emb = get_query_emb(q_norm).reshape(1, -1)
     scores, idx = index.search(q_emb, top_k)
 
     res = meta.iloc[idx[0]].copy()
@@ -197,43 +154,345 @@ def semantic_search(query: str, top_k: int) -> pd.DataFrame:
 
 
 # =========================
-# Flask app (JSON API)
+# Batch semantic highlight (FAST)
+# =========================
+def build_highlight_html_batch(
+    query_norm: str,
+    arabic_clean_list: List[str],
+    hl_topn: int,
+    seg_maxlen: int,
+) -> Tuple[List[str], Dict[str, Any]]:
+    """
+    Return list of HTML strings (one per hadith), highlighted by segment similarity.
+    Uses ONE encode() call for all segments across all hadith results (fast).
+    """
+    # If disabled:
+    if hl_topn <= 0:
+        return [escape_html(t) for t in arabic_clean_list], {"mode": "disabled"}
+
+    # Split into segments per hadith
+    per_segments: List[List[str]] = [split_ar_segments(t, seg_maxlen) for t in arabic_clean_list]
+
+    # Flatten segments
+    all_segments: List[str] = []
+    offsets: List[Tuple[int,int]] = []  # (start, end) in flattened array
+    cur = 0
+    for segs in per_segments:
+        start = cur
+        all_segments.extend(segs)
+        cur += len(segs)
+        offsets.append((start, cur))
+
+    # Edge cases
+    if len(all_segments) == 0:
+        return [escape_html(t) for t in arabic_clean_list], {"mode": "empty"}
+
+    # Encode query once + encode all segments once
+    q_emb = get_query_emb(query_norm)  # (d,)
+    seg_emb = model.encode(
+        ["passage: " + s for s in all_segments],
+        normalize_embeddings=True
+    ).astype("float32")  # (N, d)
+
+    sims_all = (seg_emb @ q_emb).astype(np.float32)  # (N,)
+
+    # Build HTML per hadith
+    html_out: List[str] = []
+    for (start, end), segs in zip(offsets, per_segments):
+        if start == end or len(segs) == 0:
+            html_out.append("")
+            continue
+
+        sims = sims_all[start:end]
+        s_min = float(np.min(sims))
+        s_max = float(np.max(sims))
+        denom = (s_max - s_min) if (s_max - s_min) > 1e-6 else 1.0
+
+        order = np.argsort(-sims)
+        keep = set(order[:min(hl_topn, len(segs))])
+
+        parts: List[str] = []
+        for i, seg in enumerate(segs):
+            w = (float(sims[i]) - s_min) / denom
+            alpha = (0.18 + 0.62 * w) if i in keep else (0.06 + 0.20 * w)
+            alpha = max(0.05, min(alpha, 0.82))
+            border_alpha = max(0.10, min(alpha * 0.8, 0.65))
+
+            style = (
+                f"background: rgba(255, 230, 120, {alpha:.3f});"
+                f"border: 1px solid rgba(234, 179, 8, {border_alpha:.3f});"
+                "border-radius: 12px;"
+                "padding: 3px 8px;"
+                "margin: 0 4px 6px 0;"
+                "display: inline;"
+            )
+            parts.append(f'<span style="{style}">{escape_html(seg)}</span> ')
+        html_out.append("".join(parts).strip())
+
+    return html_out, {"mode": "batch", "segments_total": len(all_segments)}
+
+
+# =========================
+# Flask app
 # =========================
 app = Flask(__name__)
-CORS(app)  # مهم عشان تقدر تناديه من أي هوست (HTML خارجي)
 
-@app.get("/health")
+UI_HTML = r"""
+<!doctype html>
+<html lang="ar" dir="rtl">
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <title>البحث الدلالي في الأحاديث</title>
+  <link rel="preconnect" href="https://fonts.googleapis.com">
+  <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+  <link href="https://fonts.googleapis.com/css2?family=Amiri:wght@400;700&family=Tajawal:wght@400;700&display=swap" rel="stylesheet">
+  <style>
+    :root{
+      --bg:#f6f7fb; --card:#ffffff; --text:#0f172a; --muted:#475569;
+      --line:#e5e7eb; --accent:#2563eb; --shadow: 0 10px 30px rgba(15, 23, 42, .08);
+    }
+    body{
+      margin:0; background: linear-gradient(180deg, #ffffff, var(--bg)); color: var(--text);
+      font-family: Tajawal, ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto, Arial;
+    }
+    .wrap{ max-width: 1100px; margin:0 auto; padding: 26px 16px 44px; }
+    .header{
+      background: var(--card); border: 1px solid var(--line); border-radius: 18px;
+      padding: 18px; box-shadow: var(--shadow);
+    }
+    .title{ font-family: Amiri, serif; font-size: 36px; font-weight: 700; margin:0; line-height:1.2; }
+    .sub{ margin: 6px 0 0; font-size: 18px; color: var(--muted); direction:ltr; text-align:left; }
+    .credit{ margin: 6px 0 0; font-size: 14px; color: var(--muted); direction:ltr; text-align:left; }
+
+    form{ display:flex; flex-wrap:wrap; gap:10px; align-items:center; margin-top: 14px; }
+    input[type="text"]{
+      flex: 1 1 620px; background:#fff; border:1px solid var(--line);
+      border-radius:14px; padding:14px; font-size:18px; outline:none;
+    }
+    input[type="number"]{
+      width: 92px; background:#fff; border:1px solid var(--line);
+      border-radius:14px; padding:14px 10px; font-size:16px; direction:ltr; text-align:left; outline:none;
+    }
+    button{
+      background: linear-gradient(180deg, #3b82f6, #2563eb);
+      border: 1px solid #1d4ed8; color:#fff; border-radius:14px;
+      padding:14px 16px; cursor:pointer; font-weight:700; font-size:16px;
+    }
+    button:hover{ filter: brightness(1.05); }
+
+    .controls{
+      margin-top: 12px; display:flex; gap:10px; flex-wrap:wrap; align-items:center;
+      direction:ltr; text-align:left; color: var(--muted); font-size: 13px;
+    }
+    .controls label{ display:flex; gap:8px; align-items:center; }
+    .controls input[type="range"]{ width: 200px; }
+
+    .meta{ display:flex; gap:10px; flex-wrap:wrap; margin-top: 10px; color: var(--muted); font-size: 13px; direction:ltr; }
+    .pill{ border:1px solid var(--line); background:#fff; padding:6px 10px; border-radius:999px; }
+
+    .grid{ display:grid; grid-template-columns: 1fr; gap:14px; margin-top:14px; }
+    .card{
+      background: var(--card); border: 1px solid var(--line); border-radius:18px;
+      padding: 16px; box-shadow: var(--shadow);
+    }
+    .row{ display:grid; grid-template-columns: 210px 1fr; gap:14px; }
+    @media (max-width: 900px){ .row{ grid-template-columns: 1fr; } }
+
+    .left{ color: var(--muted); font-size:14px; direction:ltr; text-align:left; }
+    .score{ font-weight:800; color: var(--accent); font-size:16px; }
+
+    .arabic{
+      direction: rtl; text-align:right; font-family: Amiri, serif; font-size:22px;
+      line-height: 2.05; background:#fbfcff; border:1px solid var(--line);
+      border-radius:16px; padding:14px; white-space: pre-wrap;
+    }
+    .english{
+      direction:ltr; text-align:left; font-size:16px; line-height:1.8; color:#111827;
+      background:#fbfcff; border:1px solid var(--line); border-radius:16px; padding:14px; white-space: pre-wrap;
+    }
+    details summary{
+      cursor:pointer; color: var(--accent); margin-top:12px; user-select:none;
+      direction:ltr; text-align:left; font-weight:700;
+    }
+    .empty{ margin-top: 14px; color: var(--muted); font-size: 15px; direction:ltr; text-align:left; }
+  </style>
+</head>
+<body>
+  <div class="wrap">
+    <div class="header">
+      <h1 class="title">البحث الدلالي في الأحاديث</h1>
+      <div class="sub">search in hadeeth (API + UI)</div>
+      <div class="credit">Done by Dr Faisal Alshargi</div>
+
+      <form id="f">
+        <input id="q" type="text" placeholder="اكتب سؤالك هنا… مثال: الاستغفار بعد الذنب وركعتين">
+        <input id="k" type="number" min="1" max="50" value="10">
+        <button type="submit">Search</button>
+      </form>
+
+      <div class="controls">
+        <label>
+          Highlight Top Segments:
+          <input id="hl" type="range" min="0" max="25" value="6">
+          <b id="hlv">6</b>
+        </label>
+        <label>
+          Segment Size:
+          <input id="seg" type="range" min="120" max="420" step="20" value="220">
+          <b id="segv">220</b>
+        </label>
+      </div>
+
+      <div id="meta" class="meta" style="display:none;"></div>
+      <div id="msg" class="empty" style="display:none;"></div>
+    </div>
+
+    <div id="grid" class="grid"></div>
+  </div>
+
+<script>
+  const $ = (id)=>document.getElementById(id);
+  function esc(s){
+    return String(s??"")
+      .replaceAll("&","&amp;").replaceAll("<","&lt;").replaceAll(">","&gt;")
+      .replaceAll('"',"&quot;").replaceAll("'","&#39;");
+  }
+  function pill(k,v){ return `<div class="pill">${esc(k)}: <b>${esc(v)}</b></div>`; }
+
+  function sync(rangeId, labelId){
+    const r=$(rangeId), l=$(labelId);
+    l.textContent = r.value;
+    r.addEventListener("input", ()=> l.textContent = r.value);
+  }
+  sync("hl","hlv"); sync("seg","segv");
+
+  $("f").addEventListener("submit", async (e)=>{
+    e.preventDefault();
+    const q = $("q").value.trim();
+    const k = parseInt($("k").value||"10",10);
+    const hl = parseInt($("hl").value||"6",10);
+    const seg = parseInt($("seg").value||"220",10);
+
+    $("msg").style.display="none";
+    $("grid").innerHTML = "";
+    $("meta").style.display="none";
+    $("meta").innerHTML = pill("Query", q) + pill("TopK", k) + pill("Highlight", hl) + pill("SegLen", seg);
+
+    if(!q){
+      $("msg").textContent="اكتب نص البحث أولًا.";
+      $("msg").style.display="block";
+      return;
+    }
+
+    $("msg").textContent="... جاري البحث";
+    $("msg").style.display="block";
+
+    const url = `/search?q=${encodeURIComponent(q)}&k=${encodeURIComponent(k)}&hl_topn=${encodeURIComponent(hl)}&seg_maxlen=${encodeURIComponent(seg)}&format=html`;
+    const t0 = performance.now();
+    const res = await fetch(url);
+    const js = await res.json();
+    const ms = Math.round(performance.now()-t0);
+
+    $("meta").style.display="flex";
+    $("meta").innerHTML =
+      pill("Rows", js.rows) + pill("Results", js.n) + pill("Time", js.took_ms ?? ms) + pill("TopK", js.k) + pill("Query", js.query);
+
+    if(!js.ok || !js.results || js.results.length===0){
+      $("msg").textContent="لا توجد نتائج. جرّب كلمات مختلفة.";
+      $("msg").style.display="block";
+      return;
+    }
+    $("msg").style.display="none";
+
+    const cards = js.results.map(r=>{
+      const ar = js.format==="html" ? (r.arabic_clean_html||esc(r.arabic_clean||"")) : esc(r.arabic_clean||"");
+      const ar_tashkeel = esc(r.arabic||"");
+      const en = esc(r.english||"");
+      return `
+        <div class="card">
+          <div class="row">
+            <div class="left">
+              <div><span class="score">${Number(r.score||0).toFixed(4)}</span> score</div>
+              <div style="margin-top:12px;">HadithID: <b>${esc(r.hadithID)}</b></div>
+              <div>Collection: <b>${esc(r.collection)}</b></div>
+              <div>No: <b>${esc(r.hadith_number)}</b></div>
+            </div>
+            <div>
+              <div class="arabic">${ar}</div>
+              <details>
+                <summary>Show Arabic with tashkeel</summary>
+                <div style="height:10px;"></div>
+                <div class="arabic">${ar_tashkeel}</div>
+              </details>
+              <details>
+                <summary>Show English</summary>
+                <div style="height:10px;"></div>
+                <div class="english">${en}</div>
+              </details>
+            </div>
+          </div>
+        </div>
+      `;
+    }).join("");
+
+    $("grid").innerHTML = cards;
+  });
+</script>
+</body>
+</html>
+"""
+
+
+@app.get("/")
 def health():
     return jsonify({
         "ok": True,
         "model": MODEL_NAME,
+        "index_ntotal": int(getattr(index, "ntotal", -1)),
         "rows": int(len(meta)),
-        "index_ntotal": int(index.ntotal),
+        "endpoints": {
+            "ui": "/ui",
+            "search_json": "/search?q=...&k=10",
+            "search_html": "/search?q=...&k=10&format=html",
+        }
     })
 
+
+@app.get("/ui")
+def ui():
+    return Response(UI_HTML, mimetype="text/html; charset=utf-8")
+
+
 @app.get("/search")
 def search():
     q = request.args.get("q", "").strip()
 
-    # k
+    # TopK
+    k_raw = request.args.get("k", str(DEFAULT_TOP_K)).strip()
     try:
-        k = int(request.args.get("k", str(DEFAULT_TOP_K)))
+        k = int(k_raw) if k_raw else DEFAULT_TOP_K
     except Exception:
         k = DEFAULT_TOP_K
-    k = min(max(1, k), MAX_TOP_K)
+    k = max(1, min(k, MAX_TOP_K))
 
-    # highlight controls
+    # Highlight controls
+    hl_raw = request.args.get("hl_topn", str(DEFAULT_HL_TOPN)).strip()
+    seg_raw = request.args.get("seg_maxlen", str(DEFAULT_SEG_MAXLEN)).strip()
     try:
-        hl_topn = int(request.args.get("hl_topn", str(DEFAULT_HL_TOPN)))
+        hl_topn = int(hl_raw) if hl_raw else DEFAULT_HL_TOPN
     except Exception:
         hl_topn = DEFAULT_HL_TOPN
-    hl_topn = min(max(1, hl_topn), MAX_HL_TOPN)
-
     try:
-        seg_maxlen = int(request.args.get("seg_maxlen", str(DEFAULT_SEG_MAXLEN)))
+        seg_maxlen = int(seg_raw) if seg_raw else DEFAULT_SEG_MAXLEN
     except Exception:
         seg_maxlen = DEFAULT_SEG_MAXLEN
-    seg_maxlen = min(max(120, seg_maxlen), MAX_SEG_MAXLEN)
+
+    hl_topn = max(0, min(hl_topn, MAX_HL_TOPN))
+    seg_maxlen = max(MIN_SEG_MAXLEN, min(seg_maxlen, MAX_SEG_MAXLEN))
+
+    fmt = (request.args.get("format", "json") or "json").lower()
+    want_html = (fmt == "html")
 
     if not q:
         return jsonify({
@@ -241,64 +500,75 @@ def search():
             "query": "",
             "query_norm": "",
             "k": k,
-            "hl_topn": hl_topn,
-            "seg_maxlen": seg_maxlen,
+            "n": 0,
+            "rows": int(len(meta)),
             "took_ms": 0,
-            "results": []
+            "format": "html" if want_html else "json",
+            "results": [],
         })
 
     t0 = time.time()
-    res_df = semantic_search(q, top_k=k)
+    df = semantic_search_df(q, top_k=k)
     took_ms = int((time.time() - t0) * 1000)
 
     q_norm = normalize_ar(q)
 
-    out: List[Dict[str, Any]] = []
-    for _, row in res_df.iterrows():
-        r = row.to_dict()
-
-        arabic_text = str(r.get("arabic", "") or "")
-        english_text = str(r.get("english", "") or "")
-
-        arabic_clean_val = r.get("arabic_clean", "")
-        if arabic_clean_val is None or (isinstance(arabic_clean_val, float) and np.isnan(arabic_clean_val)):
-            arabic_clean_val = ""
-        arabic_clean = str(arabic_clean_val).strip()
-        if not arabic_clean:
-            arabic_clean = normalize_ar(arabic_text)
-
-        # ✅ هنا الهايلايت الدلالي مثل كودك
-        arabic_highlight_html, _dbg = semantic_highlight_segments_html(
-            model=model,
+    # Build clean arabic list (fallback derive if missing)
+    arabic_list: List[str] = []
+    for _, row in df.iterrows():
+        ar = str(row.get("arabic", "") or "")
+        ar_clean = row.get("arabic_clean", "")
+        if ar_clean is None or (isinstance(ar_clean, float) and np.isnan(ar_clean)):
+            ar_clean = ""
+        ar_clean = str(ar_clean).strip()
+        if not ar_clean:
+            ar_clean = normalize_ar(ar)
+        arabic_list.append(ar_clean)
+
+    # Highlight (batch)
+    ar_html_list: List[str] = ["" for _ in arabic_list]
+    dbg: Dict[str, Any] = {}
+    if want_html:
+        ar_html_list, dbg = build_highlight_html_batch(
             query_norm=q_norm,
-            arabic_clean=arabic_clean,
-            top_n=hl_topn,
-            seg_max_len=seg_maxlen,
+            arabic_clean_list=arabic_list,
+            hl_topn=hl_topn,
+            seg_maxlen=seg_maxlen,
         )
 
-        out.append({
-            "score": float(r.get("score", 0.0)),
-            "hadithID": int(r.get("hadithID")),
-            "collection": str(r.get("collection", "")),
-            "hadith_number": int(r.get("hadith_number")),
-            "arabic": arabic_text,
-            "arabic_clean": arabic_clean,
-            "arabic_highlight_html": arabic_highlight_html,  # ✅ أهم شيء
-            "english": english_text,
-        })
+    results: List[Dict[str, Any]] = []
+    for i, (_, row) in enumerate(df.iterrows()):
+        arabic = str(row.get("arabic", "") or "")
+        english = str(row.get("english", "") or "")
+        r = {
+            "hadithID": int(row.get("hadithID")) if pd.notna(row.get("hadithID")) else None,
+            "collection": str(row.get("collection", "") or ""),
+            "hadith_number": int(row.get("hadith_number")) if pd.notna(row.get("hadith_number")) else None,
+            "score": float(row.get("score")) if pd.notna(row.get("score")) else 0.0,
+            "arabic": arabic,
+            "arabic_clean": arabic_list[i],
+            "english": english,
+        }
+        if want_html:
+            r["arabic_clean_html"] = ar_html_list[i] if ar_html_list[i] else escape_html(arabic_list[i])
+        results.append(r)
 
     return jsonify({
         "ok": True,
         "query": q,
         "query_norm": q_norm,
         "k": k,
+        "n": len(results),
+        "rows": int(len(meta)),
+        "took_ms": took_ms,
+        "format": "html" if want_html else "json",
         "hl_topn": hl_topn,
         "seg_maxlen": seg_maxlen,
-        "took_ms": took_ms,
-        "results": out
+        "debug": dbg if want_html else {},
+        "results": results,
     })
 
 
-# HF Spaces runs with gunicorn; locally:
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=7860, debug=True)
+    # local run only
+    app.run(host="127.0.0.1", port=5000, debug=True)