""" text_utils.py — Arabic text normalization, isnad filtering, negation detection, Qattan evidence extraction, and relevance scoring. Extracted from Searcher to isolate stateless text processing logic. """ import re def normalize_arabic(text: str) -> str: """Remove diacritics, normalize alef/ya/ta-marbuta.""" text = re.sub(r"[\u064B-\u065F\u0670]", "", text or "") text = text.replace("أ", "ا").replace("إ", "ا").replace("آ", "ا") text = text.replace("ى", "ي").replace("ة", "ه") text = re.sub(r"\s+", " ", text) return text.strip() def is_isnad_only(text: str, search_term: str = "") -> bool: """ Return True if text is a pure isnad chain with no critical opinion. Re-designed: a result is rejected ONLY if it's an isolated isnad. """ if not text: return True stripped = text.strip() if len(stripped) < 40: return True normalized = normalize_arabic(stripped) # Opinion verbs → not pure isnad opinion_verbs = [ "سمعت", "سألت", "سألته", "ذكر", "وذكر", "نعت", "وصف", "حكى", "روى عنه", "حكم", ] for verb in opinion_verbs: if normalize_arabic(verb) in normalized: return False # Imam names → not pure isnad imams = [ "ابن معين", "يحيى بن معين", "أحمد بن حنبل", "أبو حاتم", "أبو حاتم الرازي", "البخاري", "أبو زرعة", "الدارقطني", "ابن عدي", "ابن حبان", "يعقوب بن شيبة", "النسائي", "ابن سعد", "العقيلي", "الذهبي", "ابن حجر", "شعبة", "سفيان الثوري", "مالك بن أنس", ] for imam in imams: if normalize_arabic(imam) in normalized: return False # Pure isnad chain patterns pure_isnad_patterns = [ r"(?:حدثنا|اخبرنا|حدثني|اخبرني).*?عن.*?عن", r"عن.*?عن.*?عن.*?عن", r"(?:حدثنا|اخبرنا).*?قال\s+(?:حدثنا|اخبرنا).*?عن", r"اسناده?\s+(?:صحيح|حسن|ضعيف|واه)", r"هذا\s+(?:الاسناد|اسناد)\s+(?:صحيح|حسن|ضعيف)", ] isnad_hits = sum(1 for p in pure_isnad_patterns if re.search(p, normalized)) if isnad_hits >= 2: return True return False def check_negation(text: str, term: str) -> bool: """ Detect if a term is negated in the text. Uses a 50-char window before the term to capture Arabic negation constructions. """ normalized_text = normalize_arabic(text) normalized_term = normalize_arabic(term) # Special negation for مستقيم/يستقيم/استقام if any(x in normalized_term for x in ["مستقيم", "يستقيم", "استقام"]): for neg_phrase in [ "لا يستقيم", "لم يستقيم", "ليس بمستقيم", "غير مستقيم", "لا يصح حديثه", ]: if normalize_arabic(neg_phrase) in normalized_text: return True idx = normalized_text.find(normalized_term) if idx == -1: # Try partial match on individual words words = [w for w in normalized_term.split() if len(w) >= 3] for w in words: idx = normalized_text.find(w) if idx != -1: break if idx == -1: return False # 50-char window before the term window_start = max(0, idx - 50) before_term = normalized_text[window_start:idx].strip() negations = [ "ليس", "غير", "لا", "ما هو", "لم يكن", "ما كان", "ليس بقوي", "ليس بذاك", "ضعيف", "متروك", "لا يستحق", "لا يصح", "ليس ممن", "لا ممن", ] for neg in negations: norm_neg = normalize_arabic(neg) pattern = r"\b" + re.escape(norm_neg) + r"\b" if re.search(pattern, before_term): return True return False def has_excluded_qattan(text: str, exclude_patterns: list[str]) -> bool: """Check if text contains excluded Qattan variants (Ibn al-Qattan al-Fasi, etc.).""" normalized = normalize_arabic(text) return any(normalize_arabic(p) in normalized for p in exclude_patterns) def extract_qattan_evidence( text: str, exclude_patterns: list[str], attribution_patterns: list[str], strong_patterns: list[str], ) -> str: """ Extract Qattan attribution evidence from text. Returns the matched attribution string, or empty string if none found. """ if not text or has_excluded_qattan(text, exclude_patterns): return "" normalized = normalize_arabic(text) for pattern in attribution_patterns: normalized_pattern = normalize_arabic(pattern) match = re.search(normalized_pattern, normalized) if match: return match.group(0) for phrase in strong_patterns: normalized_phrase = normalize_arabic(phrase) if normalized_phrase in normalized: return phrase return "" def compute_relevance_score( text: str, qattan_evidence: str, narrator: str, book: str, section_book_keywords: dict, strong_patterns: list[str], ) -> float: """ Compute a relevance score (0-10) for a search result. Higher score = more likely to contain direct Qattan judgment. """ score = 0.0 # ── Qattan evidence (highest weight) ────────────────────────── if qattan_evidence: if any( p in qattan_evidence for p in ["قال يحيى", "وقال يحيى", "فقال يحيى", "يحيى بن سعيد القطان"] ): score += 4.0 # Direct quote from al-Qattan elif any(p in qattan_evidence for p in strong_patterns): score += 3.0 # Strong attribution else: score += 2.0 # Weak attribution # ── Narrator identified ────────────────────────────────────── if narrator and narrator != "أقوال عامة ونصوص متفرقة": score += 2.0 else: score += 0.5 # ── Book priority (core rijal books get highest weight) ──────── # Tier 1: Direct Jarh wa Ta'dil books (highest priority) tier1_keywords = [ "الجرح والتعديل", "الضعفاء", "الكامل", "المغني", "الميزان", "لسان الميزان", "الكاشف", "الثقات", "تقريب", "تهذيب", "الشجرة", "ديوان الضعفاء", "المجروحين", "الإصابة", "الاستيعاب", "بحر الدم", "مشاهير", "سؤالات", ] # Tier 2: Hadith sciences + comparative tier2_keywords = [ "علوم الحديث", "الكفاية", "تدريب", "فتح المغيث", "الموقظة", "الاقتراح", "نزهة النظر", "ألفية", "الباعث الحثيث", "حلية", "تاريخ الإسلام", "سير", "تاريخ بغداد", "تاريخ دمشق", "الطبقات", ] if any(kw in book for kw in tier1_keywords): score += 2.5 # Core rijal book elif any(kw in book for kw in tier2_keywords): score += 1.5 # Supporting book else: score += 0.5 # Other books # ── Text length (longer = more context) ────────────────────── if len(text) > 1000: score += 1.0 elif len(text) > 500: score += 0.5 # ── Text contains term variation ────────────────────────────── if "قال" in text and "يحيى" in text: score += 0.5 # Has attribution format return min(score, 10.0) def extract_context(text: str, term: str) -> str: """Extract a window of text around the search term for context.""" if not text or not term: return "" idx = text.find(term) if idx == -1: return text[:300] start = max(0, idx - 200) end = min(len(text), idx + len(term) + 200) return text[start:end]