Update tools/quran_search.py

tools/quran_search.py

@@ -1,60 +1,129 @@
 import requests
 import logging
 import pandas as pd
-from config import QURAN_DATA_SOURCES
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+from config import QURAN_DATA_SOURCES, MODEL_NAME, CHUNK_SIZE
 
 class QuranSearchEngine:
     def __init__(self):
-        self.api_url = "https://quranapi.pages.dev/api/"  # Correct base URL (no trailing slash needed, but consistent)
+        self.api_url = "https://quranapi.pages.dev/api/"
         self.logger = logging.getLogger(__name__)
+        self.surahs = None
+        self.all_verses = []  # List of {'surah_id': int, 'verse_num': int, 'text': str}
+        self.verse_embeddings = None
+        self.model = None
+        self._load_surahs()
+        self._load_all_verses_and_embeddings()
         
-    def get_surahs(self):
-        """Get ALL 114 surahs - guaranteed working"""
+    def _load_surahs(self):
         try:
             response = requests.get(f"{self.api_url}surah.json", timeout=5)
             response.raise_for_status()
-            surahs = response.json()  # Array of surah objects
+            self.surahs = response.json()  # List of surah dicts
+        except Exception as e:
+            self.logger.error(f"Failed to fetch surahs: {e}")
+            self.surahs = self._load_fallback_surahs()
+    
+    def _load_all_verses_and_embeddings(self):
+        """Load all verses and precompute embeddings"""
+        try:
+            for surah_id in range(1, 115):  # 1 to 114
+                response = requests.get(f"{self.api_url}{surah_id}.json", timeout=10)
+                response.raise_for_status()
+                data = response.json()
+                verses = data.get('arabic1', [])  # Arabic with tashkeel
+                for verse_num, text in enumerate(verses, start=1):
+                    self.all_verses.append({
+                        'surah_id': surah_id,
+                        'verse_num': verse_num,
+                        'text': text
+                    })
+            
+            # Precompute embeddings in chunks
+            self.model = SentenceTransformer(MODEL_NAME)
+            verse_texts = [v['text'] for v in self.all_verses]
+            self.verse_embeddings = []
+            for i in range(0, len(verse_texts), CHUNK_SIZE):
+                chunk = verse_texts[i:i + CHUNK_SIZE]
+                embeddings = self.model.encode(chunk, convert_to_tensor=False)
+                self.verse_embeddings.append(embeddings)
+            self.verse_embeddings = np.vstack(self.verse_embeddings)
+            
+        except Exception as e:
+            self.logger.error(f"Failed to load verses/embeddings: {e}")
+            # Fallback: Use limited hardcoded data (no embeddings)
+            self.all_verses = [
+                {'surah_id': 1, 'verse_num': 1, 'text': "بِسْمِ ٱللَّهِ ٱلرَّحْمَـٰنِ ٱلرَّحِيمِ"},
+                # Add more if needed, but limited
+            ]
+            self.verse_embeddings = None  # Will use keyword fallback in search
+    
+    def get_surahs(self):
+        if self.surahs:
             return [
                 (f"{s['surahNameArabicLong']} ({s['surahNameTranslation']})", i + 1)
-                for i, s in enumerate(surahs)
+                for i, s in enumerate(self.surahs)
             ]
-        except Exception as e:
-            self.logger.error(f"Failed to fetch surahs: {e}")
-            # Fallback to local Quran data
-            return self._load_fallback_surahs()
+        return self._load_fallback_surahs()
     
     def get_surah_text(self, surah_id):
-        """Get FULL surah text - tested working"""
         try:
-            response = requests.get(
-                f"{self.api_url}{surah_id}.json",
-                timeout=10
-            )
+            response = requests.get(f"{self.api_url}{surah_id}.json", timeout=10)
             response.raise_for_status()
             data = response.json()
-            verses = data['arabic1']  # Arabic with tashkeel (Uthmani-like)
-            return "\n\n".join(
-                f"آية {i + 1}: {v}"
-                for i, v in enumerate(verses)
-            )
+            verses = data['arabic1']
+            return "\n\n".join(f"آية {i + 1}: {v}" for i, v in enumerate(verses))
         except Exception as e:
             self.logger.error(f"Failed to fetch surah {surah_id}: {e}")
             return self._load_fallback_verse()
 
+    def search_verses(self, query, top_k=5):
+        """Semantic search for verses based on meaning"""
+        if self.verse_embeddings is None:
+            # Fallback to simple keyword search if embeddings failed
+            return self._keyword_fallback_search(query, top_k)
+        
+        try:
+            query_embedding = self.model.encode([query], convert_to_tensor=False)
+            similarities = cosine_similarity(query_embedding, self.verse_embeddings)[0]
+            top_indices = np.argsort(similarities)[-top_k:][::-1]
+            
+            results = []
+            for idx in top_indices:
+                verse = self.all_verses[idx]
+                surah_name = self.surahs[verse['surah_id'] - 1]['surahNameArabicLong']
+                results.append(
+                    f"سورة {surah_name} - آية {verse['verse_num']}:\n{verse['text']}\n(تشابه: {similarities[idx]:.2f})"
+                )
+            return "\n\n".join(results)
+        except Exception as e:
+            self.logger.error(f"Search failed: {e}")
+            return "حدث خطأ أثناء البحث. جرب مرة أخرى."
+
+    def _keyword_fallback_search(self, query, top_k=5):
+        """Simple keyword fallback if semantic fails"""
+        query_lower = query.lower()
+        matches = []
+        for verse in self.all_verses:
+            if query_lower in verse['text'].lower():
+                surah_name = self.surahs[verse['surah_id'] - 1]['surahNameArabicLong'] if self.surahs else f"سورة {verse['surah_id']}"
+                matches.append(f"سورة {surah_name} - آية {verse['verse_num']}:\n{verse['text']}")
+        return "\n\n".join(matches[:top_k]) or "لا توجد نتائج مطابقة."
+
+    # Existing fallback methods (unchanged)
     def _load_fallback_surahs(self):
-        """Load surah list from local CSV fallback"""
         try:
             for source in QURAN_DATA_SOURCES:
                 try:
                     df = pd.read_csv(source)
-                    # Assuming CSV has columns: surah_id, name_arabic, name_english (adjust if needed)
                     return [
                         (f"{row['name_arabic']} ({row['name_english']})", row['surah_id'])
                         for _, row in df.drop_duplicates(subset=['surah_id']).iterrows()
                     ]
                 except:
                     continue
-            # Hardcoded fallback if all sources fail
             return [
                 ("سورة الفاتحة (The Opening)", 1),
                 ("سورة البقرة (The Cow)", 2),
@@ -69,5 +138,4 @@ class QuranSearchEngine:
             ]
 
     def _load_fallback_verse(self):
-        """Load a fallback verse text"""
         return "بسم الله الرحمن الرحيم\nالله لا إله إلا هو الحي القيوم"