Upload folder using huggingface_hub

.gitattributes

@@ -1,5 +1,2 @@
 # Auto detect text files and perform LF normalization
 * text=auto
-QModel.index filter=lfs diff=lfs merge=lfs -text
-metadata.json filter=lfs diff=lfs merge=lfs -text
-models/Qwen3-32B-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -206,3 +206,7 @@ data/
 .vim/
 .DS_Store
 .vscode/
+
+QModel.index
+metadata.json
+models/Qwen3-32B-Q4_K_M.gguf

app/llm.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import re
 from typing import List
 
 from app.config import cfg
@@ -35,18 +36,32 @@ class OllamaProvider(LLMProvider):
     async def chat(
         self, messages: List[dict], temperature: float, max_tokens: int
     ) -> str:
+        # Qwen3 models return empty with think=False.  Use think=True with
+        # /no_think in the system prompt so the model responds immediately
+        # without actually producing a <think> block.
+        patched = []
+        for msg in messages:
+            if msg["role"] == "system" and "/no_think" not in msg["content"]:
+                patched.append({"role": "system", "content": msg["content"] + "\n/no_think"})
+            else:
+                patched.append(msg)
+
         loop = asyncio.get_event_loop()
         try:
             result = await loop.run_in_executor(
                 None,
                 lambda: self.client.chat(
                     model=self.model,
-                    messages=messages,
+                    messages=patched,
                     options={"temperature": temperature, "num_predict": max_tokens},
-                    think=False,
+                    think=True,
                 ),
             )
-            return result["message"]["content"].strip()
+            content = result["message"]["content"].strip()
+            # Strip any <think> blocks that slip through
+            content = re.sub(r"<think>[\s\S]*?</think>", "", content, flags=re.IGNORECASE)
+            content = re.sub(r"<think>[\s\S]*$", "", content, flags=re.IGNORECASE)
+            return content.strip()
         except Exception as exc:
             logger.error("Ollama chat failed: %s", exc)
             raise
@@ -88,7 +103,11 @@ class GGUFProvider(LLMProvider):
                     max_tokens=max_tokens,
                 ),
             )
-            return result["choices"][0]["message"]["content"].strip()
+            content = result["choices"][0]["message"]["content"] or ""
+            logger.debug("GGUF raw response (%d chars): %.500s", len(content), content)
+            # Return raw content — callers handle <think> stripping so they
+            # can still extract structured data from inside think blocks.
+            return content.strip()
         except Exception as exc:
             logger.error("GGUF chat failed: %s", exc)
             raise

app/prompts.py

@@ -11,7 +11,7 @@ from app.arabic_nlp import language_instruction
 # ═══════════════════════════════════════════════════════════════════════
 PERSONA = (
     "You are Sheikh QModel, a meticulous Islamic scholar with expertise "
-    "in Quran, Tafsir (Quranic exegesis), Hadith sciences, Fiqh, and Arabic. "
+    "in Quran, Tafsir (Quranic exegesis), Hadith sciences, and Arabic. "
     "You respond with scholarly rigor and modern clarity."
 )
 
@@ -59,15 +59,6 @@ TASK_INSTRUCTIONS: Dict[str, str] = {
         "   b. Do NOT guess or fabricate a grade.\n"
         "CRITICAL: Base authenticity ONLY on the retrieved results and collection source."
     ),
-    "fatwa": (
-        "The user seeks a religious ruling or asks about Islamic law. Steps:\n"
-        "1. Give a direct answer to the ruling question first.\n"
-        "2. Gather supporting evidence from Quran and Hadith in the results.\n"
-        "3. Quote verses and hadiths with exact references from the results.\n"
-        "4. Present scholarly reasoning based ONLY on the evidence found.\n"
-        "5. If multiple scholarly opinions exist, mention them briefly.\n"
-        "6. If the results lack sufficient evidence, state so explicitly."
-    ),
     "count": (
         "The user asks about word frequency or occurrence count. Steps:\n"
         "1. State the ANALYSIS RESULT count PROMINENTLY and FIRST.\n"

app/routers/hadith.py

@@ -13,7 +13,14 @@ from app.models import (
     HadithVerifyResponse,
     TextSearchResponse,
 )
-from app.search import hybrid_search, rewrite_query, text_search
+from app.search import (
+    filter_results_by_collection,
+    hybrid_search,
+    lookup_hadith_references,
+    merge_search_results,
+    rewrite_query,
+    text_search,
+)
 from app.state import check_ready, state
 
 router = APIRouter(prefix="/hadith", tags=["hadith"])
@@ -35,15 +42,12 @@ async def hadith_text_search(
     Use this to find a hadith when you know part of the text.
     """
     check_ready()
-    results = text_search(q, state.dataset, source_type="hadith", limit=limit)
-
-    # Optional collection filter
-    if collection:
-        col_lower = collection.lower()
-        results = [
-            r for r in results
-            if col_lower in (r.get("collection", "") or r.get("reference", "")).lower()
-        ]
+    direct_results = lookup_hadith_references(q, state.dataset, collection=collection, limit=limit)
+    text_results = text_search(q, state.dataset, source_type="hadith", limit=limit)
+    results = filter_results_by_collection(
+        merge_search_results(direct_results, text_results, limit=limit),
+        collection,
+    )
 
     # If text search returns few results, augment with semantic search
     if len(results) < 3:
@@ -53,20 +57,8 @@ async def hadith_text_search(
             state.embed_model, state.faiss_index, state.dataset,
             top_n=limit, source_type="hadith",
         )
-        seen_ids = {r.get("id") for r in results}
-        for sr in sem_results:
-            if sr.get("id") not in seen_ids:
-                results.append(sr)
-                seen_ids.add(sr.get("id"))
-        results = sorted(results, key=lambda x: x.get("_score", 0), reverse=True)[:limit]
-
-    # Optional collection filter
-    if collection:
-        col_lower = collection.lower()
-        results = [
-            r for r in results
-            if col_lower in (r.get("collection", "") or r.get("reference", "")).lower()
-        ]
+        sem_results = filter_results_by_collection(sem_results, collection)
+        results = merge_search_results(results, sem_results, limit=limit)
 
     return TextSearchResponse(
         query=q,
@@ -143,14 +135,12 @@ async def verify_hadith(
     check_ready()
     t0 = time.perf_counter()
 
-    # 1. Try text search first for exact matches
-    text_results = text_search(q, state.dataset, source_type="hadith", limit=10)
-    if collection:
-        col_lower = collection.lower()
-        text_results = [
-            r for r in text_results
-            if col_lower in (r.get("collection", "") or r.get("reference", "")).lower()
-        ]
+    # 1. Try direct reference and text search first for exact matches
+    direct_results = lookup_hadith_references(q, state.dataset, collection=collection, limit=10)
+    text_results = filter_results_by_collection(
+        text_search(q, state.dataset, source_type="hadith", limit=10),
+        collection,
+    )
 
     # 2. Also try semantic search with auth intent for better matching
     rewrite = await rewrite_query(q, state.llm)
@@ -160,16 +150,18 @@ async def verify_hadith(
         state.embed_model, state.faiss_index, state.dataset,
         top_n=10, source_type="hadith",
     )
+    semantic_results = filter_results_by_collection(semantic_results, collection)
+    merged_results = merge_search_results(direct_results, text_results, semantic_results, limit=10)
 
     # 3. Pick best result — prefer high-confidence text matches,
     #    then semantic results, then lower-confidence text matches
     best = None
-    if text_results and text_results[0].get("_score", 0) > 2.0:
-        best = text_results[0]
-    elif semantic_results:
-        best = semantic_results[0]
-    elif text_results:
+    if direct_results:
+        best = direct_results[0]
+    elif text_results and text_results[0].get("_score", 0) > 2.0:
         best = text_results[0]
+    elif merged_results:
+        best = merged_results[0]
 
     if best:
         return HadithVerifyResponse(

app/routers/quran.py

@@ -19,7 +19,7 @@ from app.models import (
     VerseItem,
     WordFrequencyResponse,
 )
-from app.search import hybrid_search, rewrite_query, text_search
+from app.search import hybrid_search, lookup_quran_verses, merge_search_results, rewrite_query, text_search
 from app.state import check_ready, state
 
 router = APIRouter(prefix="/quran", tags=["quran"])
@@ -39,7 +39,19 @@ async def quran_text_search(
     Use this to find a verse when you know part of the text.
     """
     check_ready()
-    results = text_search(q, state.dataset, source_type="quran", limit=limit)
+    direct_results = lookup_quran_verses(q, state.dataset, limit=limit)
+    text_results = text_search(q, state.dataset, source_type="quran", limit=limit)
+    results = merge_search_results(direct_results, text_results, limit=limit)
+
+    if len(results) < min(limit, 3):
+        rewrite = await rewrite_query(q, state.llm)
+        semantic_results = await hybrid_search(
+            q, rewrite,
+            state.embed_model, state.faiss_index, state.dataset,
+            top_n=limit, source_type="quran",
+        )
+        results = merge_search_results(results, semantic_results, limit=limit)
+
     return TextSearchResponse(
         query=q,
         count=len(results),
@@ -73,12 +85,14 @@ async def quran_topic_search(
     Finds verses about a topic even when the exact words don't appear (e.g. "patience", "charity").
     """
     check_ready()
+    direct_results = lookup_quran_verses(topic, state.dataset, limit=top_k)
     rewrite = await rewrite_query(topic, state.llm)
-    results = await hybrid_search(
+    semantic_results = await hybrid_search(
         topic, rewrite,
         state.embed_model, state.faiss_index, state.dataset,
         top_n=top_k, source_type="quran",
     )
+    results = merge_search_results(direct_results, semantic_results, limit=top_k)
     return TextSearchResponse(
         query=topic,
         count=len(results),

app/search.py

@@ -6,6 +6,7 @@ import json
 import logging
 import re
 from collections import Counter
+from itertools import chain
 from difflib import SequenceMatcher
 from typing import Dict, List, Literal, Optional
 
@@ -33,7 +34,7 @@ Reply ONLY with a valid JSON object — no markdown, no preamble:
   "ar_query": "<query in clear Arabic فصحى, ≤25 words>",
   "en_query": "<query in clear English, ≤25 words>",
   "keywords": ["<3-7 key Arabic or English terms from the question>"],
-  "intent": "<one of: fatwa | tafsir | hadith | count | surah_info | auth | general>"
+    "intent": "<one of: tafsir | hadith | count | surah_info | auth | general>"
 }
 
 Intent Detection Rules (CRITICAL):
@@ -52,7 +53,6 @@ Intent Detection Rules (CRITICAL):
   (كم مرة ذُكرت كلمة, how many times is word X mentioned, عدد مرات ذكر كلمة)
   NOTE: "كم عدد آيات سورة" is surah_info NOT count!
   IMPORTANT: The word being counted MUST be the first keyword.
-- 'fatwa' intent = asking for a religious ruling (ما حكم, is X halal/haram, حلال أم حرام)
 - 'general' intent = other Islamic questions
 
 Rewriting Rules:
@@ -76,44 +76,413 @@ Examples:
 - "ما معنى حديث إنما الأعمال" → intent: hadith, ar_query: "إنما الأعمال"
 - "ابحث عن حديث عن الصبر" → intent: hadith, keywords: ["صبر", "الصبر", "patience"]
 - "find hadith about fasting" → intent: hadith, keywords: ["صيام", "صوم", "fasting"]
-- "ما حكم الربا في الإسلام" → intent: fatwa, keywords: ["ربا", "الربا", "usury"]
+- "ما حكم الربا في الإسلام" → intent: general, keywords: ["ربا", "الربا", "usury"]
 - "هل الحديث ده صحيح: من كان يؤمن بالله" → intent: auth, ar_query: "من كان يؤمن بالله"
 """
 
 
+_QURAN_REF_NUMERIC = re.compile(r"\b(\d{1,3})\s*:\s*(\d{1,3})\b")
+_QURAN_REF_AR_NAME_FIRST = re.compile(
+    r"(?:سورة|سوره)\s+([\u0621-\u06FF\u0750-\u077F\s]+?)"
+    r"(?:\s+(?:الآية|آية|الايه|ايه)\s*|[\s,:،-]+)(\d{1,3})",
+)
+_QURAN_REF_AR_VERSE_FIRST = re.compile(
+    r"(?:الآية|آية|الايه|ايه)\s*(\d{1,3})\s+(?:من|في)\s+(?:سورة|سوره)\s+([\u0621-\u06FF\u0750-\u077F\s]+)",
+)
+_QURAN_REF_EN_NAME_FIRST = re.compile(
+    r"(?:surah|sura)\s+([A-Za-z'\- ]+?)"
+    r"(?:\s+(?:ayah|verse|ayat)\s*|[\s,:-]+)(\d{1,3})",
+    re.I,
+)
+_QURAN_REF_EN_VERSE_FIRST = re.compile(
+    r"(?:ayah|verse|ayat)\s*(\d{1,3})\s+(?:of|in)\s+(?:surah|sura)\s+([A-Za-z'\- ]+)",
+    re.I,
+)
+
+_COLLECTION_ALIASES = {
+    "sahih al-bukhari": "Sahih al-Bukhari",
+    "sahih bukhari": "Sahih al-Bukhari",
+    "al bukhari": "Sahih al-Bukhari",
+    "bukhari": "Sahih al-Bukhari",
+    "البخاري": "Sahih al-Bukhari",
+    "صحيح البخاري": "Sahih al-Bukhari",
+    "sahih muslim": "Sahih Muslim",
+    "muslim": "Sahih Muslim",
+    "مسلم": "Sahih Muslim",
+    "صحيح مسلم": "Sahih Muslim",
+    "sunan an nasai": "Sunan an-Nasai",
+    "sunan an-nasai": "Sunan an-Nasai",
+    "nasai": "Sunan an-Nasai",
+    "nasa'i": "Sunan an-Nasai",
+    "نسائي": "Sunan an-Nasai",
+    "النسائي": "Sunan an-Nasai",
+    "sunan abu dawood": "Sunan Abu Dawood",
+    "sunan abu dawood": "Sunan Abu Dawood",
+    "abu dawood": "Sunan Abu Dawood",
+    "abu dawood": "Sunan Abu Dawood",
+    "أبو داود": "Sunan Abu Dawood",
+    "ابو داود": "Sunan Abu Dawood",
+    "jami at tirmidhi": "Jami' at-Tirmidhi",
+    "jami at-tirmidhi": "Jami' at-Tirmidhi",
+    "tirmidhi": "Jami' at-Tirmidhi",
+    "الترمذي": "Jami' at-Tirmidhi",
+    "ترمذي": "Jami' at-Tirmidhi",
+    "sunan ibn majah": "Sunan Ibn Majah",
+    "ibn majah": "Sunan Ibn Majah",
+    "ابن ماجه": "Sunan Ibn Majah",
+    "sunan al darimi": "Sunan al-Darimi",
+    "sunan al-darimi": "Sunan al-Darimi",
+    "darimi": "Sunan al-Darimi",
+    "الدارمي": "Sunan al-Darimi",
+    "muwatta malik": "Muwatta Malik",
+    "muwatta": "Muwatta Malik",
+    "موطأ مالك": "Muwatta Malik",
+    "موطا مالك": "Muwatta Malik",
+    "malik": "Muwatta Malik",
+    "musnad ahmad": "Musnad Ahmad",
+    "ahmad": "Musnad Ahmad",
+    "ahmed": "Musnad Ahmad",
+    "أحمد": "Musnad Ahmad",
+    "مسند أحمد": "Musnad Ahmad",
+}
+_SORTED_COLLECTION_ALIASES = sorted(_COLLECTION_ALIASES.items(), key=lambda item: len(item[0]), reverse=True)
+
+
+def _find_balanced_json(text: str) -> Optional[str]:
+    """Find the first balanced {...} in *text*; return it or None."""
+    start = text.find("{")
+    if start == -1:
+        return None
+    depth = 0
+    in_string = False
+    escaped = False
+    for idx in range(start, len(text)):
+        ch = text[idx]
+        if escaped:
+            escaped = False
+            continue
+        if ch == "\\":
+            escaped = True
+            continue
+        if ch == '"':
+            in_string = not in_string
+            continue
+        if in_string:
+            continue
+        if ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start:idx + 1]
+    return None
+
+
+def _extract_json_object(text: str) -> Optional[str]:
+    """Extract the first balanced JSON object from model output."""
+    if not text:
+        return None
+
+    # Remove common wrappers some models add around structured responses.
+    cleaned = re.sub(r"```(?:json)?", "", text, flags=re.IGNORECASE)
+    cleaned = cleaned.replace("```", "")
+    # Strip closed <think> blocks
+    cleaned = re.sub(r"<think>[\s\S]*?</think>", "", cleaned, flags=re.IGNORECASE)
+    cleaned = cleaned.strip()
+
+    # Before stripping unclosed <think> (which removes everything after it),
+    # check if a JSON object exists anywhere in the remaining text —
+    # the model may have emitted JSON inside a truncated think block.
+    if "{" not in cleaned:
+        # No JSON at all, strip unclosed think and give up
+        cleaned = re.sub(r"<think>[\s\S]*$", "", cleaned, flags=re.IGNORECASE).strip()
+        if not cleaned:
+            return None
+    else:
+        # Try to extract JSON first; only strip unclosed <think> if that fails
+        candidate = _find_balanced_json(cleaned)
+        if candidate:
+            return candidate
+        cleaned = re.sub(r"<think>[\s\S]*$", "", cleaned, flags=re.IGNORECASE).strip()
+        if not cleaned:
+            return None
+
+    if cleaned.startswith("{") and cleaned.endswith("}"):
+        return cleaned
+
+    return _find_balanced_json(cleaned)
+
+
+def _detect_intent_regex(query: str) -> str:
+    """Detect intent from raw query using regex when LLM rewrite is unavailable."""
+    # surah_info: asking about surah metadata (verse count, type, etc.)
+    if re.search(
+        r"كم\s+(?:عدد\s+)?آيات?|عدد\s+آيات?|كم\s+آية|how many\s+verses?|number of\s+verses?",
+        query, re.I,
+    ):
+        return "surah_info"
+    if re.search(
+        r"(?:هل|ما\s+نوع)\s+(?:سورة|سوره)\s+.+\s+(?:مكية|مدنية)",
+        query,
+    ):
+        return "surah_info"
+
+    # count: word frequency
+    if re.search(
+        r"كم مرة|كم تكرر|عدد مرات|تكرار|كم ذُكر|كم وردت?",
+        query,
+    ):
+        return "count"
+    if re.search(
+        r"\b(how many times?|count|frequency|occurrences? of)\b",
+        query, re.I,
+    ):
+        return "count"
+
+    # auth: hadith authenticity check
+    if re.search(
+        r"صحيح[؟?]|هل صحيح|درجة الحديث|هل هذا حديث|is.+authentic|verify hadith|hadith.+grade",
+        query, re.I,
+    ):
+        return "auth"
+
+    # hadith: hadith lookup (check before tafsir since both can match Arabic text)
+    if re.search(
+        r"حديث\s+عن|ابحث عن حديث|ما معنى حديث|find hadith|hadith about",
+        query, re.I,
+    ):
+        return "hadith"
+
+    # tafsir: Quranic verse lookup — Arabic verse text or explicit tafsir request
+    if re.search(
+        r"ابحث عن آية|ما تفسير|تفسير آية|آية عن|الآية التي|find verse|verse about|tafsir",
+        query, re.I,
+    ):
+        return "tafsir"
+
+    # If query contains substantial Arabic with Quranic markers (diacritics, special chars),
+    # treat as tafsir (verse text lookup)
+    ar_chars = len(re.findall(r"[\u0600-\u06FF]", query))
+    diacritics = len(re.findall(r"[\u064B-\u0655\u0670\u06D6-\u06ED\u06E1-\u06E9\u0610-\u061A]", query))
+    if ar_chars > 10 and diacritics >= 3:
+        return "tafsir"
+
+    return "general"
+
+
 async def rewrite_query(raw: str, llm: LLMProvider) -> Dict:
     """Rewrite query for better retrieval."""
     cached = await rewrite_cache.get(raw)
     if cached:
         return cached
 
+    detected_intent = _detect_intent_regex(raw)
     fallback = {
         "ar_query": normalize_arabic(raw),
         "en_query": raw,
-        "keywords": raw.split()[:7],
-        "intent":   "general",
+        "keywords": [light_stem(t) for t in tokenize_ar(raw)][:7],
+        "intent":   detected_intent,
     }
     try:
-        text = await llm.chat(
-            messages=[
-                {"role": "system", "content": _REWRITE_SYSTEM},
-                {"role": "user",   "content": raw},
-            ],
-            max_tokens=220,
-            temperature=0.0,
-        )
-        text   = re.sub(r"```(?:json)?\n?|\n?```", "", text).strip()
-        result = json.loads(text)
+        messages = [
+            {"role": "system", "content": _REWRITE_SYSTEM},
+            {"role": "user",   "content": raw},
+        ]
+        text = ""
+        for _attempt, temp in enumerate((0.0, 0.3)):
+            text = await llm.chat(
+                messages=messages, max_tokens=1024, temperature=temp,
+            )
+            if text.strip():
+                break
+            logger.warning("Empty rewrite response (attempt %d), retrying with temperature=%.1f",
+                           _attempt + 1, 0.3)
+        logger.debug("Raw rewrite response (%d chars): %.300s", len(text), text)
+        json_payload = _extract_json_object(text)
+        if not json_payload:
+            raise ValueError(
+                f"Model did not return a JSON object (got {len(text)} chars: {text[:120]!r})"
+            )
+        result = json.loads(json_payload)
         for k in ("ar_query", "en_query", "keywords", "intent"):
             result.setdefault(k, fallback[k])
+
+        if not isinstance(result.get("keywords"), list):
+            result["keywords"] = fallback["keywords"]
+        else:
+            result["keywords"] = [str(x).strip() for x in result["keywords"] if str(x).strip()][:7]
+            if not result["keywords"]:
+                result["keywords"] = fallback["keywords"]
+
+        result["intent"] = str(result.get("intent") or fallback["intent"]).strip().lower()
+        if result["intent"] == "fatwa":
+            result["intent"] = "general"
+        if result["intent"] not in {"tafsir", "hadith", "count", "surah_info", "auth", "general"}:
+            result["intent"] = fallback["intent"]
+
+        result["ar_query"] = str(result.get("ar_query") or fallback["ar_query"]).strip()[:400]
+        result["en_query"] = str(result.get("en_query") or fallback["en_query"]).strip()[:400]
+
         await rewrite_cache.set(result, raw)
         logger.info("Rewrite: intent=%s ar=%s", result["intent"], result["ar_query"][:60])
         return result
     except Exception as exc:
-        logger.warning("Query rewrite failed (%s) — using fallback", exc)
+        logger.warning("Query rewrite failed (%s) — using fallback (intent=%s)", exc, fallback["intent"])
         return fallback
 
 
+def result_key(item: dict) -> tuple:
+    """Build a stable key for deduplicating search results."""
+    item_type = item.get("type", "")
+    if item_type == "quran":
+        return (
+            "quran",
+            int(item.get("surah_number") or 0),
+            int(item.get("ayah_number") or item.get("verse_number") or 0),
+        )
+    if item_type == "hadith":
+        return (
+            "hadith",
+            normalize_arabic(item.get("collection", ""), aggressive=True).lower(),
+            int(item.get("hadith_number") or 0),
+            normalize_arabic(item.get("reference", ""), aggressive=True).lower(),
+        )
+    return (
+        item_type,
+        normalize_arabic(item.get("source") or item.get("reference", ""), aggressive=True).lower(),
+        normalize_arabic(item.get("arabic", "")[:80], aggressive=True).lower(),
+        item.get("english", "")[:80].lower(),
+    )
+
+
+def merge_search_results(*result_groups: list, limit: Optional[int] = None) -> list:
+    """Merge multiple ranked result groups, deduplicating by stable content key."""
+    merged: dict[tuple, dict] = {}
+    for item in chain.from_iterable(result_groups):
+        key = result_key(item)
+        current = merged.get(key)
+        if current is None or item.get("_score", 0.0) > current.get("_score", 0.0):
+            merged[key] = item
+
+    results = sorted(merged.values(), key=lambda row: row.get("_score", 0.0), reverse=True)
+    return results[:limit] if limit is not None else results
+
+
+def normalize_collection_name(text: str) -> Optional[str]:
+    """Resolve a collection alias to the canonical dataset collection name."""
+    if not text:
+        return None
+
+    normalized = normalize_arabic(text, aggressive=True).lower()
+    normalized = normalized.replace("_", " ")
+    normalized = re.sub(r"[^a-z0-9\u0600-\u06FF\s'\-]+", " ", normalized)
+    normalized = re.sub(r"\s+", " ", normalized).strip()
+
+    for alias, canonical in _SORTED_COLLECTION_ALIASES:
+        if alias in normalized:
+            return canonical
+    return None
+
+
+def filter_results_by_collection(results: list, collection: Optional[str]) -> list:
+    """Filter hadith results by canonical or fuzzy collection name."""
+    if not collection:
+        return list(results)
+
+    canonical = normalize_collection_name(collection)
+    collection_norm = normalize_arabic(collection, aggressive=True).lower().strip()
+    filtered = []
+    for item in results:
+        haystack = normalize_arabic(
+            f"{item.get('collection', '')} {item.get('reference', '')}",
+            aggressive=True,
+        ).lower()
+        if canonical and item.get("collection", "") == canonical:
+            filtered.append(item)
+            continue
+        if collection_norm and collection_norm in haystack:
+            filtered.append(item)
+    return filtered
+
+
+def _surah_matches(item: dict, surah_query: str) -> bool:
+    query_norm = normalize_arabic(surah_query, aggressive=True).lower().strip()
+    query_clean = re.sub(r"^(ال|al[\-\s']*)", "", query_norm, flags=re.I).strip()
+
+    for field in ("surah_name_ar", "surah_name_en", "surah_name_transliteration"):
+        value = item.get(field, "")
+        if not value:
+            continue
+        value_norm = normalize_arabic(value, aggressive=True).lower().strip()
+        value_clean = re.sub(r"^(ال|al[\-\s']*)", "", value_norm, flags=re.I).strip()
+        if query_norm == value_norm or query_clean == value_clean:
+            return True
+        if query_clean and value_clean and (query_clean in value_clean or value_clean in query_clean):
+            return True
+    return False
+
+
+def lookup_quran_verses(query: str, dataset: list, limit: int = 5) -> list:
+    """Resolve direct Quran references like 2:255 or Surah Al-Baqarah 255."""
+    if not query:
+        return []
+
+    matches = []
+    numeric = _QURAN_REF_NUMERIC.search(query)
+    if numeric:
+        surah_num, verse_num = int(numeric.group(1)), int(numeric.group(2))
+        for item in dataset:
+            if item.get("type") != "quran":
+                continue
+            if item.get("surah_number") == surah_num and (item.get("ayah_number") or item.get("verse_number")) == verse_num:
+                matches.append({**item, "_score": 9.5})
+                return matches
+
+    named_patterns = (
+        (_QURAN_REF_AR_NAME_FIRST, lambda m: (m.group(1), int(m.group(2)))),
+        (_QURAN_REF_AR_VERSE_FIRST, lambda m: (m.group(2), int(m.group(1)))),
+        (_QURAN_REF_EN_NAME_FIRST, lambda m: (m.group(1), int(m.group(2)))),
+        (_QURAN_REF_EN_VERSE_FIRST, lambda m: (m.group(2), int(m.group(1)))),
+    )
+    for pattern, extractor in named_patterns:
+        match = pattern.search(query)
+        if not match:
+            continue
+        surah_query, verse_num = extractor(match)
+        for item in dataset:
+            if item.get("type") != "quran":
+                continue
+            if _surah_matches(item, surah_query) and (item.get("ayah_number") or item.get("verse_number")) == verse_num:
+                matches.append({**item, "_score": 9.0})
+        if matches:
+            break
+
+    return matches[:limit]
+
+
+def lookup_hadith_references(query: str, dataset: list, collection: Optional[str] = None, limit: int = 5) -> list:
+    """Resolve direct hadith references like Bukhari 1 or مسلم 1907."""
+    if not query and not collection:
+        return []
+
+    canonical_collection = normalize_collection_name(collection or "") or normalize_collection_name(query)
+    number_match = re.search(r"\b(\d{1,5})\b", query)
+    if not canonical_collection or not number_match:
+        return []
+
+    hadith_number = int(number_match.group(1))
+    matches = []
+    for item in dataset:
+        if item.get("type") != "hadith":
+            continue
+        if item.get("collection") != canonical_collection:
+            continue
+        if int(item.get("hadith_number") or 0) != hadith_number:
+            continue
+        matches.append({**item, "_score": 9.0})
+    return matches[:limit]
+
+
 # ═══════════════════════════════════════════════════════════════════════
 # BM25 SCORING
 # ═══════════════════════════════════════════════════════════════════════

app/state.py

@@ -24,11 +24,131 @@ from app.arabic_nlp import detect_language, normalize_arabic
 from app.config import cfg
 from app.llm import LLMProvider, get_llm_provider
 from app.prompts import build_messages, not_found_answer
-from app.search import build_context, hybrid_search, rewrite_query, text_search
+from app.search import (
+    build_context,
+    hybrid_search,
+    lookup_hadith_references,
+    lookup_quran_verses,
+    merge_search_results,
+    rewrite_query,
+    text_search,
+)
 
 logger = logging.getLogger("qmodel.state")
 
 
+# ═══════════════════════════════════════════════════════════════════════
+# SURAH INFO PROGRAMMATIC FALLBACK
+# ═══════════════════════════════════════════════════════════════════════
+def _results_fallback(results: list, lang: str) -> str:
+    """Generate a direct answer from search results when LLM returns empty."""
+    if not results:
+        return not_found_answer(lang)
+
+    top = results[0]
+    item_type = top.get("type", "")
+
+    if item_type == "quran":
+        ar_text = top.get("arabic", "")
+        en_text = top.get("english", "")
+        surah_ar = top.get("surah_name_ar", "")
+        surah_en = top.get("surah_name_en", "")
+        surah_num = top.get("surah_number", "")
+        verse_num = top.get("verse_number", "")
+        tafsir = top.get("tafsir_ar", "") if lang == "arabic" else top.get("tafsir_en", "")
+
+        if lang == "arabic":
+            lines = [
+                f"هذه الآية الكريمة من سورة {surah_ar} ({surah_en})، الآية {verse_num}.",
+                "",
+                f"┌─────────────────────────────────────────────┐",
+                f"│  ❝ {ar_text} ❞",
+                f"│  📝 Translation: {en_text}",
+                f"│  📖 Source: سورة {surah_ar} ({surah_en}) | رقم السورة: {surah_num} | الآية: {verse_num}",
+                f"└─────────────────────────────────────────────┘",
+            ]
+            if tafsir:
+                lines.append(f"\n{tafsir}")
+            lines.append("\nوالله أعلم.")
+            return "\n".join(lines)
+        else:
+            lines = [
+                f"This noble verse is from Surah {surah_en} ({surah_ar}), verse {verse_num}.",
+                "",
+                f"┌─────────────────────────────────────────────┐",
+                f"│  ❝ {ar_text} ❞",
+                f"│  📝 Translation: {en_text}",
+                f"│  📖 Source: Surah {surah_en} ({surah_ar}) | Surah Number: {surah_num} | Verse: {verse_num}",
+                f"└─────────────────────────────────────────────┘",
+            ]
+            if tafsir:
+                lines.append(f"\n{tafsir}")
+            lines.append("\nAnd Allah knows best.")
+            return "\n".join(lines)
+
+    elif item_type == "hadith":
+        ar_text = top.get("arabic", "")
+        en_text = top.get("english", "")
+        source = top.get("source") or top.get("reference", "")
+        grade = top.get("grade", "")
+        grade_str = f" [{grade}]" if grade else ""
+
+        if lang == "arabic":
+            lines = [
+                f"الحديث الشريف{grade_str}:",
+                "",
+                f"┌─────────────────────────────────────────────┐",
+                f"│  ❝ {ar_text} ❞",
+                f"│  📝 Translation: {en_text}",
+                f"│  📖 Source: {source}",
+                f"└─────────────────────────────────────────────┘",
+                "\nوالله أعلم.",
+            ]
+            return "\n".join(lines)
+        else:
+            lines = [
+                f"The Hadith{grade_str}:",
+                "",
+                f"┌─────────────────────────────────────────────┐",
+                f"│  ❝ {ar_text} ❞",
+                f"│  📝 Translation: {en_text}",
+                f"│  📖 Source: {source}",
+                f"└─────────────────────────────────────────────┘",
+                "\nAnd Allah knows best.",
+            ]
+            return "\n".join(lines)
+
+    return not_found_answer(lang)
+
+
+def _surah_info_fallback(info: dict, lang: str) -> str:
+    """Generate a direct answer from surah metadata when LLM fails."""
+    name_ar = info.get("surah_name_ar", "")
+    name_en = info.get("surah_name_en", "")
+    number = info.get("surah_number", "")
+    verses = info.get("total_verses", "")
+    rev = info.get("revelation_type", "")
+    translit = info.get("surah_name_transliteration", "")
+
+    rev_ar = "مكية" if rev == "meccan" else "مدنية" if rev == "medinan" else rev
+    rev_en = rev.capitalize() if rev else ""
+
+    if lang == "arabic":
+        return (
+            f"سورة {name_ar} ({translit}) هي السورة رقم {number} في القرآن الكريم.\n"
+            f"عدد آياتها: {verses} آية.\n"
+            f"نوعها: {rev_ar}.\n"
+            f"والله أعلم."
+        )
+    return (
+        f"Surah {name_en} ({translit} / {name_ar}) is surah number {number} "
+        f"in the Holy Quran.\n"
+        f"Total verses: {verses}.\n"
+        f"Revelation type: {rev_en}.\n"
+        f"And Allah knows best."
+    )
+
+
 # ═══════════════════════════════════════════════════════════════════════
 # POST-GENERATION HALLUCINATION CHECK
 # ═══════════════════════════════════════════════════════════════════════
@@ -278,6 +398,32 @@ def _verify_surah_info(answer: str, surah_info: dict) -> str:
             flags=re.I,
         )
 
+    # ── Fix wrong surah numbers ─────────────────────────────────────
+    if correct_number is not None:
+        def _fix_surah_number(m: re.Match) -> str:
+            num = int(m.group(2))
+            if num == correct_number:
+                return m.group(0)
+            logger.warning(
+                "Surah info hallucination: surah number %d -> correcting to %d",
+                num, correct_number,
+            )
+            return m.group(1) + " " + str(correct_number)
+
+        # Arabic: "رقم X" / "رقمها X" / "ترتيبها X"
+        answer = re.sub(
+            r"(رقم[ها]*|ترتيب[ها]*)\s+(\d+)",
+            _fix_surah_number,
+            answer,
+        )
+        # English: "surah number X", "number X"
+        answer = re.sub(
+            r"((?:surah\s+)?number)\s+(\d+)",
+            _fix_surah_number,
+            answer,
+            flags=re.I,
+        )
+
     # ── Fix wrong verse counts ──────────────────────────────────────
     if correct_verses is not None:
         def _fix_verse_count(m: re.Match) -> str:
@@ -432,10 +578,24 @@ async def run_rag_pipeline(
         surah_task, kw_task, search_task,
     )
 
-    # 2b. Text search fallback — catches exact matches missed by FAISS
+    # 2b. Direct reference lookup — catches Quran/Hadith references early.
+    direct_queries = list(dict.fromkeys([
+        question,
+        rewrite.get("ar_query", ""),
+        rewrite.get("en_query", ""),
+    ]))
+    direct_results = []
+    if source_type in (None, "quran"):
+        for query in direct_queries:
+            direct_results.extend(lookup_quran_verses(query, state.dataset, limit=top_k))
+    if source_type in (None, "hadith"):
+        for query in direct_queries:
+            direct_results.extend(lookup_hadith_references(query, state.dataset, limit=top_k))
+    results = merge_search_results(direct_results, results, limit=top_k)
+
+    # 2c. Text search fallback — catches exact matches missed by FAISS.
     #     For auth/hadith/tafsir intents, also search with the rewritten ar_query
     #     which should contain the actual text fragment to look up.
-    seen_ids = {r.get("id") for r in results}
     ar_q = rewrite.get("ar_query", "")
 
     # Determine text search source filter based on intent
@@ -446,19 +606,16 @@ async def run_rag_pipeline(
         text_src = "hadith"
 
     text_limit = top_k * 2 if intent in ("auth", "hadith", "tafsir") else top_k
+    text_results = []
     for q in dict.fromkeys([ar_q, question]):  # deduplicated, ar_query first
         if not q:
             continue
         for hit in text_search(q, state.dataset, text_src, limit=text_limit):
-            if hit.get("id") not in seen_ids:
-                # Boost text search hits for auth intent (exact text match is crucial)
-                if intent == "auth" and hit.get("_score", 0) > 2.0:
-                    hit["_score"] = hit["_score"] + 1.0
-                results.append(hit)
-                seen_ids.add(hit.get("id"))
-    if len(results) > top_k:
-        results.sort(key=lambda x: x.get("_score", 0), reverse=True)
-        results = results[:top_k]
+            # Boost text search hits for auth intent (exact text match is crucial)
+            if intent == "auth" and hit.get("_score", 0) > 2.0:
+                hit = {**hit, "_score": hit["_score"] + 1.0}
+            text_results.append(hit)
+    results = merge_search_results(results, text_results, limit=top_k)
 
     # 3a. Surah metadata lookup
     surah_info = None
@@ -505,6 +662,24 @@ async def run_rag_pipeline(
             "latency_ms": int((time.perf_counter() - t0) * 1000),
         }
 
+    # 5b. Surah metadata: deterministic answer (faster & more reliable than LLM)
+    if surah_info and intent == "surah_info":
+        answer = _surah_info_fallback(surah_info, lang)
+        latency = int((time.perf_counter() - t0) * 1000)
+        logger.info(
+            "Pipeline done (surah_info deterministic) | lang=%s | %d ms",
+            lang, latency,
+        )
+        return {
+            "answer":     answer,
+            "language":   lang,
+            "intent":     intent,
+            "analysis":   None,
+            "sources":    results,
+            "top_score":  top_score,
+            "latency_ms": latency,
+        }
+
     # 6. Build context + prompt + LLM call
     context  = build_context(results)
     messages = build_messages(context, question, lang, intent, analysis, surah_info)
@@ -515,6 +690,19 @@ async def run_rag_pipeline(
             max_tokens=cfg.MAX_TOKENS,
             temperature=cfg.TEMPERATURE,
         )
+        # Strip residual <think> blocks that Qwen3 may emit
+        answer = re.sub(r"<think>[\s\S]*?</think>", "", answer, flags=re.IGNORECASE)
+        answer = re.sub(r"<think>[\s\S]*$", "", answer, flags=re.IGNORECASE)
+        answer = answer.strip()
+
+        if not answer:
+            logger.warning("LLM returned empty answer — using results fallback")
+            if surah_info:
+                answer = _surah_info_fallback(surah_info, lang)
+            elif results:
+                answer = _results_fallback(results, lang)
+            else:
+                answer = not_found_answer(lang)
     except Exception as exc:
         logger.error("LLM call failed: %s", exc)
         raise HTTPException(status_code=502, detail="LLM service unavailable")