Update app.py

app.py

@@ -8,7 +8,6 @@ from typing import List, Dict, Any, Tuple
 import numpy as np
 import pandas as pd
 import faiss
-
 from flask import Flask, request, jsonify
 from flask_cors import CORS
 from sentence_transformers import SentenceTransformer
@@ -24,10 +23,10 @@ MODEL_NAME = os.getenv("HADITH_MODEL_NAME", "intfloat/multilingual-e5-base")
 DEFAULT_TOP_K = 10
 MAX_TOP_K = 50
 
-DEFAULT_HL_TOPN = 6       # segments with strong highlight
+DEFAULT_HL_TOPN = 6
 MAX_HL_TOPN = 25
 
-DEFAULT_SEG_MAXLEN = 220  # segment size
+DEFAULT_SEG_MAXLEN = 220
 MAX_SEG_MAXLEN = 420
 
 
@@ -42,7 +41,6 @@ _AR_DIACRITICS = re.compile(r"""
 """, re.VERBOSE)
 
 def normalize_ar(text: str) -> str:
-    """Remove tashkeel + normalize common Arabic letter variants."""
     if text is None:
         return ""
     text = str(text)
@@ -69,15 +67,12 @@ def escape_html(s: str) -> str:
 
 
 # =========================
-# Semantic segment highlighting
+# Segmenting + semantic highlight
 # =========================
-def split_ar_segments(text: str, max_len: int = DEFAULT_SEG_MAXLEN) -> List[str]:
-    """Split Arabic clean text into short segments for semantic highlighting."""
+def split_ar_segments(text: str, max_len: int) -> List[str]:
     if not text:
         return []
     t = re.sub(r"\s+", " ", str(text)).strip()
-
-    # Split on punctuation (Arabic + Latin)
     parts = re.split(r"(?<=[\.\!\?؟\،\,\;\:])\s+", t)
 
     segs: List[str] = []
@@ -96,7 +91,6 @@ def split_ar_segments(text: str, max_len: int = DEFAULT_SEG_MAXLEN) -> List[str]
     if buf:
         segs.append(buf)
 
-    # Fallback chunking
     if len(segs) <= 1 and len(t) > max_len:
         segs = [t[i:i+max_len].strip() for i in range(0, len(t), max_len) if t[i:i+max_len].strip()]
 
@@ -106,18 +100,18 @@ def semantic_highlight_segments_html(
     model: SentenceTransformer,
     query_norm: str,
     arabic_clean: str,
-    top_n: int = DEFAULT_HL_TOPN,
-    seg_max_len: int = DEFAULT_SEG_MAXLEN
-) -> Tuple[str, float, float]:
+    top_n: int,
+    seg_max_len: int
+) -> Tuple[str, List[Dict[str, Any]]]:
     """
-    Returns HTML with segments colored by semantic similarity to query.
-    Also returns min/max similarity.
+    Returns:
+      - HTML string with highlighted segments
+      - segments_debug: list of {seg, sim, strong}
     """
     segs = split_ar_segments(arabic_clean, max_len=seg_max_len)
     if not segs:
-        return escape_html(arabic_clean), 0.0, 0.0
+        return escape_html(arabic_clean), []
 
-    # E5 format: "query:" and "passage:"
     q_emb = model.encode(["query: " + query_norm], normalize_embeddings=True).astype("float32")
     seg_emb = model.encode(["passage: " + s for s in segs], normalize_embeddings=True).astype("float32")
 
@@ -130,11 +124,14 @@ def semantic_highlight_segments_html(
     keep = set(order[:min(top_n, len(segs))])
 
     html_parts: List[str] = []
+    dbg: List[Dict[str, Any]] = []
+
     for i, seg in enumerate(segs):
         w = (float(sims[i]) - s_min) / denom  # 0..1
+        strong = i in keep
 
-        # Strong highlight for closest segments, softer for others
-        alpha = (0.18 + 0.62 * w) if i in keep else (0.06 + 0.20 * w)
+        # Strong highlight for top segments, softer for others
+        alpha = (0.18 + 0.62 * w) if strong else (0.06 + 0.20 * w)
         alpha = max(0.05, min(alpha, 0.82))
         border_alpha = max(0.10, min(alpha * 0.8, 0.65))
 
@@ -147,12 +144,13 @@ def semantic_highlight_segments_html(
             "display: inline;"
         )
         html_parts.append(f'<span style="{style}">{escape_html(seg)}</span> ')
+        dbg.append({"seg": seg, "sim": float(sims[i]), "strong": bool(strong)})
 
     html = "".join(html_parts).strip()
     if not html:
         html = escape_html(arabic_clean)
 
-    return html, s_min, s_max
+    return html, dbg
 
 
 # =========================
@@ -164,14 +162,9 @@ if not os.path.exists(INDEX_PATH):
 if not os.path.exists(META_PATH):
     raise FileNotFoundError(f"Meta parquet not found: {META_PATH}")
 
-print(f"[BOOT] Loading model: {MODEL_NAME}")
 model = SentenceTransformer(MODEL_NAME)
-
-print(f"[BOOT] Loading faiss index: {INDEX_PATH}")
 index = faiss.read_index(INDEX_PATH)
-
-print(f"[BOOT] Loading meta: {META_PATH}")
-meta = pd.read_parquet(META_PATH)
+meta  = pd.read_parquet(META_PATH)
 
 required_cols = {"hadithID", "collection", "hadith_number", "arabic", "english"}
 missing = required_cols - set(meta.columns)
@@ -181,52 +174,33 @@ if missing:
 if "arabic_clean" not in meta.columns:
     meta["arabic_clean"] = ""
 
-# normalize types lightly
-meta["arabic"] = meta["arabic"].fillna("").astype(str)
-meta["english"] = meta["english"].fillna("").astype(str)
-meta["collection"] = meta["collection"].fillna("").astype(str)
-
 
-def semantic_search_df(query: str, top_k: int) -> pd.DataFrame:
+def semantic_search(query: str, top_k: int) -> pd.DataFrame:
     q = str(query or "").strip()
     if not q:
         return meta.iloc[0:0].copy()
 
     top_k = max(1, min(int(top_k), MAX_TOP_K))
-    q_norm = normalize_ar(q)
 
+    q_norm = normalize_ar(q)
     q_emb = model.encode(["query: " + q_norm], normalize_embeddings=True).astype("float32")
+
     scores, idx = index.search(q_emb, top_k)
 
     res = meta.iloc[idx[0]].copy()
     res["score"] = scores[0]
     res = res.sort_values("score", ascending=False)
 
-    # filter empty arabic rows (avoid empty cards)
     res["arabic"] = res["arabic"].fillna("").astype(str)
     res = res[res["arabic"].str.strip() != ""]
-
     return res
 
 
 # =========================
-# Flask API
+# Flask app (JSON API)
 # =========================
 app = Flask(__name__)
-CORS(app, resources={r"/*": {"origins": "*"}})
-
-
-@app.get("/")
-def root():
-    return jsonify({
-        "ok": True,
-        "service": "hadith semantic search",
-        "endpoints": {
-            "health": "/health",
-            "search": "/search?q=...&k=10&hl_topn=6&seg_maxlen=220"
-        }
-    })
-
+CORS(app)  # مهم عشان تقدر تناديه من أي هوست (HTML خارجي)
 
 @app.get("/health")
 def health():
@@ -234,34 +208,29 @@ def health():
         "ok": True,
         "model": MODEL_NAME,
         "rows": int(len(meta)),
-        "index_ntotal": int(getattr(index, "ntotal", -1)),
+        "index_ntotal": int(index.ntotal),
     })
 
-
 @app.get("/search")
 def search():
     q = request.args.get("q", "").strip()
 
-    # topK
-    k_raw = request.args.get("k", str(DEFAULT_TOP_K)).strip()
+    # k
     try:
-        k_int = int(k_raw) if k_raw else DEFAULT_TOP_K
+        k = int(request.args.get("k", str(DEFAULT_TOP_K)))
     except Exception:
-        k_int = DEFAULT_TOP_K
-    k_int = min(max(1, k_int), MAX_TOP_K)
-
-    # highlight knobs
-    hl_raw = request.args.get("hl_topn", str(DEFAULT_HL_TOPN)).strip()
-    seg_raw = request.args.get("seg_maxlen", str(DEFAULT_SEG_MAXLEN)).strip()
+        k = DEFAULT_TOP_K
+    k = min(max(1, k), MAX_TOP_K)
 
+    # highlight controls
     try:
-        hl_topn = int(hl_raw) if hl_raw else DEFAULT_HL_TOPN
+        hl_topn = int(request.args.get("hl_topn", str(DEFAULT_HL_TOPN)))
     except Exception:
         hl_topn = DEFAULT_HL_TOPN
     hl_topn = min(max(1, hl_topn), MAX_HL_TOPN)
 
     try:
-        seg_maxlen = int(seg_raw) if seg_raw else DEFAULT_SEG_MAXLEN
+        seg_maxlen = int(request.args.get("seg_maxlen", str(DEFAULT_SEG_MAXLEN)))
     except Exception:
         seg_maxlen = DEFAULT_SEG_MAXLEN
     seg_maxlen = min(max(120, seg_maxlen), MAX_SEG_MAXLEN)
@@ -271,14 +240,15 @@ def search():
             "ok": True,
             "query": "",
             "query_norm": "",
-            "k": k_int,
-            "rows": int(len(meta)),
+            "k": k,
+            "hl_topn": hl_topn,
+            "seg_maxlen": seg_maxlen,
             "took_ms": 0,
-            "results": [],
+            "results": []
         })
 
     t0 = time.time()
-    res_df = semantic_search_df(q, top_k=k_int)
+    res_df = semantic_search(q, top_k=k)
     took_ms = int((time.time() - t0) * 1000)
 
     q_norm = normalize_ar(q)
@@ -287,66 +257,48 @@ def search():
     for _, row in res_df.iterrows():
         r = row.to_dict()
 
-        arabic = str(r.get("arabic", "") or "")
-        english = str(r.get("english", "") or "")
+        arabic_text = str(r.get("arabic", "") or "")
+        english_text = str(r.get("english", "") or "")
 
         arabic_clean_val = r.get("arabic_clean", "")
-        if arabic_clean_val is None:
-            arabic_clean_val = ""
-        # handle NaN
-        if isinstance(arabic_clean_val, float) and np.isnan(arabic_clean_val):
+        if arabic_clean_val is None or (isinstance(arabic_clean_val, float) and np.isnan(arabic_clean_val)):
             arabic_clean_val = ""
         arabic_clean = str(arabic_clean_val).strip()
         if not arabic_clean:
-            arabic_clean = normalize_ar(arabic)
+            arabic_clean = normalize_ar(arabic_text)
 
-        # ✅ semantic highlight segments (returns HTML spans)
-        arabic_clean_html, s_min, s_max = semantic_highlight_segments_html(
+        # ✅ هنا الهايلايت الدلالي مثل كودك
+        arabic_highlight_html, _dbg = semantic_highlight_segments_html(
             model=model,
             query_norm=q_norm,
             arabic_clean=arabic_clean,
             top_n=hl_topn,
-            seg_max_len=seg_maxlen
+            seg_max_len=seg_maxlen,
         )
 
-        # final fallback never empty
-        if not str(arabic_clean_html).strip():
-            arabic_clean_html = escape_html(arabic_clean if arabic_clean else arabic)
-
         out.append({
+            "score": float(r.get("score", 0.0)),
             "hadithID": int(r.get("hadithID")),
             "collection": str(r.get("collection", "")),
             "hadith_number": int(r.get("hadith_number")),
-            "score": float(r.get("score", 0.0)),
-
-            "arabic": arabic,
+            "arabic": arabic_text,
             "arabic_clean": arabic_clean,
-            "english": english,
-
-            # HTML-ready fields
-            "arabic_clean_html": arabic_clean_html,
-            "arabic_html": escape_html(arabic),
-            "english_html": escape_html(english),
-
-            # optional stats
-            "hl_min": float(s_min),
-            "hl_max": float(s_max),
+            "arabic_highlight_html": arabic_highlight_html,  # ✅ أهم شيء
+            "english": english_text,
         })
 
     return jsonify({
         "ok": True,
         "query": q,
         "query_norm": q_norm,
-        "k": k_int,
+        "k": k,
         "hl_topn": hl_topn,
         "seg_maxlen": seg_maxlen,
-        "rows": int(len(meta)),
         "took_ms": took_ms,
-        "results": out,
+        "results": out
     })
 
 
-# HuggingFace Docker runs via CMD (gunicorn/uvicorn) عادة
-# لكن هذا مفيد لو شغّلته محلياً:
+# HF Spaces runs with gunicorn; locally:
 if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=int(os.getenv("PORT", "7860")), debug=True)
+    app.run(host="0.0.0.0", port=7860, debug=True)