Update app.py

app.py

@@ -3,11 +3,12 @@ from __future__ import annotations
 import os
 import re
 import time
-from typing import Any, Dict, List, Optional, Tuple
+from typing import List, Dict, Any, Tuple
 
 import numpy as np
 import pandas as pd
 import faiss
+
 from flask import Flask, request, jsonify
 from flask_cors import CORS
 from sentence_transformers import SentenceTransformer
@@ -23,9 +24,15 @@ MODEL_NAME = os.getenv("HADITH_MODEL_NAME", "intfloat/multilingual-e5-base")
 DEFAULT_TOP_K = 10
 MAX_TOP_K = 50
 
+DEFAULT_HL_TOPN = 6       # segments with strong highlight
+MAX_HL_TOPN = 25
+
+DEFAULT_SEG_MAXLEN = 220  # segment size
+MAX_SEG_MAXLEN = 420
+
 
 # =========================
-# Arabic normalization (remove tashkeel + normalize letters)
+# Arabic normalization
 # =========================
 _AR_DIACRITICS = re.compile(r"""
     [\u0610-\u061A]
@@ -35,6 +42,7 @@ _AR_DIACRITICS = re.compile(r"""
 """, re.VERBOSE)
 
 def normalize_ar(text: str) -> str:
+    """Remove tashkeel + normalize common Arabic letter variants."""
     if text is None:
         return ""
     text = str(text)
@@ -47,99 +55,160 @@ def normalize_ar(text: str) -> str:
     text = re.sub(r"\s+", " ", text).strip()
     return text
 
+def escape_html(s: str) -> str:
+    if s is None:
+        return ""
+    return (
+        str(s)
+        .replace("&", "&")
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace('"', "&quot;")
+        .replace("'", "&#39;")
+    )
+
 
 # =========================
-# Lazy load (load resources on demand)
+# Semantic segment highlighting
 # =========================
-_model: Optional[SentenceTransformer] = None
-_index = None
-_meta: Optional[pd.DataFrame] = None
-
-def get_resources() -> Tuple[SentenceTransformer, Any, pd.DataFrame]:
-    global _model, _index, _meta
+def split_ar_segments(text: str, max_len: int = DEFAULT_SEG_MAXLEN) -> List[str]:
+    """Split Arabic clean text into short segments for semantic highlighting."""
+    if not text:
+        return []
+    t = re.sub(r"\s+", " ", str(text)).strip()
+
+    # Split on punctuation (Arabic + Latin)
+    parts = re.split(r"(?<=[\.\!\?؟\،\,\;\:])\s+", t)
+
+    segs: List[str] = []
+    buf = ""
+    for p in parts:
+        p = (p or "").strip()
+        if not p:
+            continue
+        if not buf:
+            buf = p
+        elif len(buf) + 1 + len(p) <= max_len:
+            buf = f"{buf} {p}"
+        else:
+            segs.append(buf)
+            buf = p
+    if buf:
+        segs.append(buf)
+
+    # Fallback chunking
+    if len(segs) <= 1 and len(t) > max_len:
+        segs = [t[i:i+max_len].strip() for i in range(0, len(t), max_len) if t[i:i+max_len].strip()]
+
+    return segs
+
+def semantic_highlight_segments_html(
+    model: SentenceTransformer,
+    query_norm: str,
+    arabic_clean: str,
+    top_n: int = DEFAULT_HL_TOPN,
+    seg_max_len: int = DEFAULT_SEG_MAXLEN
+) -> Tuple[str, float, float]:
+    """
+    Returns HTML with segments colored by semantic similarity to query.
+    Also returns min/max similarity.
+    """
+    segs = split_ar_segments(arabic_clean, max_len=seg_max_len)
+    if not segs:
+        return escape_html(arabic_clean), 0.0, 0.0
 
-    if _model is not None and _index is not None and _meta is not None:
-        return _model, _index, _meta
+    # E5 format: "query:" and "passage:"
+    q_emb = model.encode(["query: " + query_norm], normalize_embeddings=True).astype("float32")
+    seg_emb = model.encode(["passage: " + s for s in segs], normalize_embeddings=True).astype("float32")
 
-    if not os.path.exists(INDEX_PATH):
-        raise FileNotFoundError(f"FAISS index not found: {INDEX_PATH}")
+    sims = (seg_emb @ q_emb[0]).astype(np.float32)
+    s_min = float(np.min(sims))
+    s_max = float(np.max(sims))
+    denom = (s_max - s_min) if (s_max - s_min) > 1e-6 else 1.0
 
-    if not os.path.exists(META_PATH):
-        raise FileNotFoundError(f"Meta parquet not found: {META_PATH}")
+    order = np.argsort(-sims)
+    keep = set(order[:min(top_n, len(segs))])
 
-    _model = SentenceTransformer(MODEL_NAME)
-    _index = faiss.read_index(INDEX_PATH)
-    _meta  = pd.read_parquet(META_PATH)
+    html_parts: List[str] = []
+    for i, seg in enumerate(segs):
+        w = (float(sims[i]) - s_min) / denom  # 0..1
 
-    required_cols = {"hadithID", "collection", "hadith_number", "arabic", "english"}
-    missing = required_cols - set(_meta.columns)
-    if missing:
-        raise ValueError(f"Meta is missing required columns: {missing}")
+        # Strong highlight for closest segments, softer for others
+        alpha = (0.18 + 0.62 * w) if i in keep else (0.06 + 0.20 * w)
+        alpha = max(0.05, min(alpha, 0.82))
+        border_alpha = max(0.10, min(alpha * 0.8, 0.65))
 
-    if "arabic_clean" not in _meta.columns:
-        _meta["arabic_clean"] = ""
+        style = (
+            f"background: rgba(255, 230, 120, {alpha:.3f});"
+            f"border: 1px solid rgba(234, 179, 8, {border_alpha:.3f});"
+            "border-radius: 12px;"
+            "padding: 3px 8px;"
+            "margin: 0 4px 6px 0;"
+            "display: inline;"
+        )
+        html_parts.append(f'<span style="{style}">{escape_html(seg)}</span> ')
 
-    # Normalize types / fill missing
-    for col in ["arabic", "english", "arabic_clean", "collection"]:
-        if col in _meta.columns:
-            _meta[col] = _meta[col].fillna("").astype(str)
+    html = "".join(html_parts).strip()
+    if not html:
+        html = escape_html(arabic_clean)
 
-    return _model, _index, _meta
+    return html, s_min, s_max
 
 
 # =========================
-# Search
+# Load model + index + meta (once)
 # =========================
-def semantic_search(query: str, top_k: int = DEFAULT_TOP_K) -> pd.DataFrame:
-    model, index, meta = get_resources()
+if not os.path.exists(INDEX_PATH):
+    raise FileNotFoundError(f"FAISS index not found: {INDEX_PATH}")
+
+if not os.path.exists(META_PATH):
+    raise FileNotFoundError(f"Meta parquet not found: {META_PATH}")
 
+print(f"[BOOT] Loading model: {MODEL_NAME}")
+model = SentenceTransformer(MODEL_NAME)
+
+print(f"[BOOT] Loading faiss index: {INDEX_PATH}")
+index = faiss.read_index(INDEX_PATH)
+
+print(f"[BOOT] Loading meta: {META_PATH}")
+meta = pd.read_parquet(META_PATH)
+
+required_cols = {"hadithID", "collection", "hadith_number", "arabic", "english"}
+missing = required_cols - set(meta.columns)
+if missing:
+    raise ValueError(f"Meta is missing required columns: {missing}")
+
+if "arabic_clean" not in meta.columns:
+    meta["arabic_clean"] = ""
+
+# normalize types lightly
+meta["arabic"] = meta["arabic"].fillna("").astype(str)
+meta["english"] = meta["english"].fillna("").astype(str)
+meta["collection"] = meta["collection"].fillna("").astype(str)
+
+
+def semantic_search_df(query: str, top_k: int) -> pd.DataFrame:
     q = str(query or "").strip()
     if not q:
         return meta.iloc[0:0].copy()
 
     top_k = max(1, min(int(top_k), MAX_TOP_K))
-
     q_norm = normalize_ar(q)
-    q_emb = model.encode(["query: " + q_norm], normalize_embeddings=True).astype("float32")
 
+    q_emb = model.encode(["query: " + q_norm], normalize_embeddings=True).astype("float32")
     scores, idx = index.search(q_emb, top_k)
 
     res = meta.iloc[idx[0]].copy()
-    res["score"] = scores[0].astype(float)
+    res["score"] = scores[0]
     res = res.sort_values("score", ascending=False)
 
-    # Filter empty arabic just in case
+    # filter empty arabic rows (avoid empty cards)
     res["arabic"] = res["arabic"].fillna("").astype(str)
     res = res[res["arabic"].str.strip() != ""]
 
     return res
 
 
-def row_to_json(row: pd.Series, include_text: bool = True) -> Dict[str, Any]:
-    arabic = str(row.get("arabic", "") or "")
-    english = str(row.get("english", "") or "")
-
-    arabic_clean = str(row.get("arabic_clean", "") or "").strip()
-    if not arabic_clean:
-        arabic_clean = normalize_ar(arabic)
-
-    base: Dict[str, Any] = {
-        "score": float(row.get("score", 0.0)),
-        "hadithID": int(row.get("hadithID")),
-        "collection": str(row.get("collection", "")),
-        "hadith_number": int(row.get("hadith_number")),
-    }
-
-    if include_text:
-        base.update({
-            "arabic": arabic,
-            "arabic_clean": arabic_clean,
-            "english": english,
-        })
-
-    return base
-
-
 # =========================
 # Flask API
 # =========================
@@ -151,122 +220,133 @@ CORS(app, resources={r"/*": {"origins": "*"}})
 def root():
     return jsonify({
         "ok": True,
-        "service": "hadeeth semantic search api",
-        "endpoints": ["/health", "/search (GET/POST)"]
+        "service": "hadith semantic search",
+        "endpoints": {
+            "health": "/health",
+            "search": "/search?q=...&k=10&hl_topn=6&seg_maxlen=220"
+        }
     })
 
 
 @app.get("/health")
 def health():
-    # Don't force-load model/index/meta here if you want it super fast
-    # But we can still show file presence:
-    files_ok = os.path.exists(INDEX_PATH) and os.path.exists(META_PATH)
-    info = {
+    return jsonify({
         "ok": True,
-        "files_ok": files_ok,
-        "index_path": INDEX_PATH,
-        "meta_path": META_PATH,
         "model": MODEL_NAME,
-    }
+        "rows": int(len(meta)),
+        "index_ntotal": int(getattr(index, "ntotal", -1)),
+    })
 
-    # If you want to show counts (this will load resources):
-    try:
-        _, index, meta = get_resources()
-        info["rows"] = int(len(meta))
-        info["index_ntotal"] = int(getattr(index, "ntotal", -1))
-        info["loaded"] = True
-    except Exception as e:
-        info["loaded"] = False
-        info["load_error"] = str(e)
 
-    return jsonify(info)
+@app.get("/search")
+def search():
+    q = request.args.get("q", "").strip()
 
+    # topK
+    k_raw = request.args.get("k", str(DEFAULT_TOP_K)).strip()
+    try:
+        k_int = int(k_raw) if k_raw else DEFAULT_TOP_K
+    except Exception:
+        k_int = DEFAULT_TOP_K
+    k_int = min(max(1, k_int), MAX_TOP_K)
 
-@app.post("/search")
-def search_post():
-    """
-    Body JSON:
-    {
-      "q": "��لرزق",
-      "k": 10,
-      "include_text": true
-    }
-    """
-    payload = request.get_json(silent=True) or {}
+    # highlight knobs
+    hl_raw = request.args.get("hl_topn", str(DEFAULT_HL_TOPN)).strip()
+    seg_raw = request.args.get("seg_maxlen", str(DEFAULT_SEG_MAXLEN)).strip()
 
-    q = (payload.get("q") or "").strip()
-    if not q:
-        return jsonify({"ok": False, "error": "Missing 'q'"}), 400
+    try:
+        hl_topn = int(hl_raw) if hl_raw else DEFAULT_HL_TOPN
+    except Exception:
+        hl_topn = DEFAULT_HL_TOPN
+    hl_topn = min(max(1, hl_topn), MAX_HL_TOPN)
 
-    k = payload.get("k", DEFAULT_TOP_K)
     try:
-        k = int(k)
+        seg_maxlen = int(seg_raw) if seg_raw else DEFAULT_SEG_MAXLEN
     except Exception:
-        k = DEFAULT_TOP_K
-    k = max(1, min(k, MAX_TOP_K))
+        seg_maxlen = DEFAULT_SEG_MAXLEN
+    seg_maxlen = min(max(120, seg_maxlen), MAX_SEG_MAXLEN)
 
-    include_text = payload.get("include_text", True)
-    include_text = bool(include_text)
+    if not q:
+        return jsonify({
+            "ok": True,
+            "query": "",
+            "query_norm": "",
+            "k": k_int,
+            "rows": int(len(meta)),
+            "took_ms": 0,
+            "results": [],
+        })
 
     t0 = time.time()
-    try:
-        res_df = semantic_search(q, top_k=k)
-    except Exception as e:
-        return jsonify({"ok": False, "error": str(e)}), 500
+    res_df = semantic_search_df(q, top_k=k_int)
     took_ms = int((time.time() - t0) * 1000)
 
-    results = [row_to_json(r, include_text=include_text) for _, r in res_df.iterrows()]
-
-    return jsonify({
-        "ok": True,
-        "query": q,
-        "query_norm": normalize_ar(q),
-        "k": k,
-        "took_ms": took_ms,
-        "results_count": len(results),
-        "results": results
-    })
-
-
-@app.get("/search")
-def search_get():
-    """
-    GET /search?q=...&k=10&include_text=1
-    """
-    q = (request.args.get("q") or "").strip()
-    if not q:
-        return jsonify({"ok": False, "error": "Missing 'q'"}), 400
+    q_norm = normalize_ar(q)
 
-    k_raw = request.args.get("k", str(DEFAULT_TOP_K))
-    try:
-        k = int(k_raw)
-    except Exception:
-        k = DEFAULT_TOP_K
-    k = max(1, min(k, MAX_TOP_K))
+    out: List[Dict[str, Any]] = []
+    for _, row in res_df.iterrows():
+        r = row.to_dict()
+
+        arabic = str(r.get("arabic", "") or "")
+        english = str(r.get("english", "") or "")
+
+        arabic_clean_val = r.get("arabic_clean", "")
+        if arabic_clean_val is None:
+            arabic_clean_val = ""
+        # handle NaN
+        if isinstance(arabic_clean_val, float) and np.isnan(arabic_clean_val):
+            arabic_clean_val = ""
+        arabic_clean = str(arabic_clean_val).strip()
+        if not arabic_clean:
+            arabic_clean = normalize_ar(arabic)
+
+        # ✅ semantic highlight segments (returns HTML spans)
+        arabic_clean_html, s_min, s_max = semantic_highlight_segments_html(
+            model=model,
+            query_norm=q_norm,
+            arabic_clean=arabic_clean,
+            top_n=hl_topn,
+            seg_max_len=seg_maxlen
+        )
+
+        # final fallback never empty
+        if not str(arabic_clean_html).strip():
+            arabic_clean_html = escape_html(arabic_clean if arabic_clean else arabic)
+
+        out.append({
+            "hadithID": int(r.get("hadithID")),
+            "collection": str(r.get("collection", "")),
+            "hadith_number": int(r.get("hadith_number")),
+            "score": float(r.get("score", 0.0)),
 
-    include_text_raw = request.args.get("include_text", "1")
-    include_text = include_text_raw not in ("0", "false", "False", "")
+            "arabic": arabic,
+            "arabic_clean": arabic_clean,
+            "english": english,
 
-    t0 = time.time()
-    try:
-        res_df = semantic_search(q, top_k=k)
-    except Exception as e:
-        return jsonify({"ok": False, "error": str(e)}), 500
-    took_ms = int((time.time() - t0) * 1000)
+            # HTML-ready fields
+            "arabic_clean_html": arabic_clean_html,
+            "arabic_html": escape_html(arabic),
+            "english_html": escape_html(english),
 
-    results = [row_to_json(r, include_text=include_text) for _, r in res_df.iterrows()]
+            # optional stats
+            "hl_min": float(s_min),
+            "hl_max": float(s_max),
+        })
 
     return jsonify({
         "ok": True,
         "query": q,
-        "query_norm": normalize_ar(q),
-        "k": k,
+        "query_norm": q_norm,
+        "k": k_int,
+        "hl_topn": hl_topn,
+        "seg_maxlen": seg_maxlen,
+        "rows": int(len(meta)),
         "took_ms": took_ms,
-        "results_count": len(results),
-        "results": results
+        "results": out,
     })
 
 
+# HuggingFace Docker runs via CMD (gunicorn/uvicorn) عادة
+# لكن هذا مفيد لو شغّلته محلياً:
 if __name__ == "__main__":
-    # Local dev only
-    app.run(host="0.0.0.0", port=7860, debug=False)
+    app.run(host="0.0.0.0", port=int(os.getenv("PORT", "7860")), debug=True)