Update tools/quran_search.py

tools/quran_search.py

@@ -5,6 +5,7 @@ from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 from config import QURAN_DATA_SOURCES, MODEL_NAME, CHUNK_SIZE
 import time
+import sys
 
 class QuranSearchEngine:
     def __init__(self):
@@ -14,28 +15,32 @@ class QuranSearchEngine:
         self.all_verses = []  # List of {'surah_id': int, 'verse_num': int, 'text': str}
         self.verse_embeddings = None
         self.model = None
-        print("Starting QuranSearchEngine initialization...")  # Debug
-        self._load_full_quran()
-        print(f"Surahs loaded: {len(self.surahs) if self.surahs else 0}")  # Debug
-        self._load_all_verses_and_embeddings()
-        print(f"Verses loaded: {len(self.all_verses)}")  # Debug
-        
+        print("Starting QuranSearchEngine initialization at", time.ctime(), file=sys.stderr)  # Debug to stderr
+        try:
+            self._load_full_quran()
+            print(f"Surahs loaded: {len(self.surahs) if self.surahs else 0}", file=sys.stderr)  # Debug
+            self._load_all_verses_and_embeddings()
+            print(f"Verses loaded: {len(self.all_verses)}", file=sys.stderr)  # Debug
+        except Exception as e:
+            self.logger.error(f"Initialization failed: {e}", exc_info=True)
+            print(f"Initialization error: {e}", file=sys.stderr)
+            self._load_fallback_data()  # Ensure minimal startup
+
     def _load_full_quran(self):
-        max_retries = 3
+        max_retries = 5  # Increased retries
         for attempt in range(max_retries):
             try:
-                response = requests.get(f"{self.api_url}surah.json", timeout=10)
+                response = requests.get(f"{self.api_url}surah.json", timeout=15)  # Increased timeout
                 response.raise_for_status()
                 self.surahs = response.json()
-                # Add 'id' to surahs for consistency
                 for i, s in enumerate(self.surahs):
                     s['id'] = i + 1
-                # Fetch full verses
+                self.all_verses = []  # Reset verses
                 for surah_id in range(1, 115):
-                    surah_response = requests.get(f"{self.api_url}{surah_id}.json", timeout=10)
+                    surah_response = requests.get(f"{self.api_url}{surah_id}.json", timeout=15)
                     surah_response.raise_for_status()
                     data = surah_response.json()
-                    verses = data['arabic1']  # Arabic with tashkeel
+                    verses = data['arabic1']
                     for verse_num, text in enumerate(verses, start=1):
                         self.all_verses.append({
                             'surah_id': surah_id,
@@ -47,22 +52,24 @@ class QuranSearchEngine:
                 self.logger.error(f"Attempt {attempt + 1}/{max_retries} failed to fetch Quran data: {e}")
                 if attempt == max_retries - 1:
                     self._load_fallback_data()
-                time.sleep(2 ** attempt)
-    
+                time.sleep(2 ** attempt)  # Exponential backoff
+
     def _load_fallback_data(self):
+        self.logger.warning("Falling back to minimal data due to API failure")
         self.surahs = self._load_fallback_surahs()
         self.all_verses = [
             {'surah_id': 1, 'verse_num': 1, 'text': "بِسْمِ ٱللَّهِ ٱلرَّحْمَـٰنِ ٱلرَّحِيمِ"},
             {'surah_id': 1, 'verse_num': 2, 'text': "ٱلْحَمْدُ لِلَّهِ رَبِّ ٱلْعَٰلَمِينَ"}
-            # Add more hardcoded verses if needed, but limited
         ]
-    
+
     def _load_all_verses_and_embeddings(self):
         if not self.all_verses:
             return
         
         try:
+            print("Attempting to load model...", file=sys.stderr)  # Debug
             self.model = SentenceTransformer(MODEL_NAME)
+            print("Model loaded successfully", file=sys.stderr)  # Debug
             verse_texts = [v['text'] for v in self.all_verses]
             self.verse_embeddings = []
             for i in range(0, len(verse_texts), CHUNK_SIZE):
@@ -70,23 +77,22 @@ class QuranSearchEngine:
                 embeddings = self.model.encode(chunk, convert_to_tensor=False)
                 self.verse_embeddings.append(embeddings)
             self.verse_embeddings = np.vstack(self.verse_embeddings)
+            print("Embeddings computed successfully", file=sys.stderr)  # Debug
         except Exception as e:
-            self.logger.error(f"Failed to compute embeddings: {e}")
+            self.logger.error(f"Failed to compute embeddings: {e}", exc_info=True)
             self.verse_embeddings = None
-    
+            self.logger.warning("Falling back to keyword-based search due to embedding failure")
+
     def get_surahs(self):
         if self.surahs:
-            return [
-                (s['surahNameArabicLong'], s['id'])
-                for s in self.surahs
-            ]
+            return [(s['surahNameArabicLong'], s['id']) for s in self.surahs]
         return self._load_fallback_surahs()
-    
+
     def get_surah_text(self, surah_id):
         max_retries = 3
         for attempt in range(max_retries):
             try:
-                response = requests.get(f"{self.api_url}{surah_id}.json", timeout=10)
+                response = requests.get(f"{self.api_url}{surah_id}.json", timeout=15)
                 response.raise_for_status()
                 data = response.json()
                 verses = data['arabic1']
@@ -96,12 +102,13 @@ class QuranSearchEngine:
                 if attempt == max_retries - 1:
                     return self._load_fallback_verse()
                 time.sleep(2 ** attempt)
-    
+
     def search_verses(self, query, top_k=5):
         if self.verse_embeddings is None or not self.all_verses:
             return self._keyword_fallback_search(query, top_k)
         
         try:
+            print(f"Encoding query: {query}", file=sys.stderr)  # Debug
             query_embedding = self.model.encode([query], convert_to_tensor=False)
             similarities = cosine_similarity(query_embedding, self.verse_embeddings)[0]
             top_indices = np.argsort(similarities)[-top_k:][::-1]
@@ -110,12 +117,10 @@ class QuranSearchEngine:
             for idx in top_indices:
                 verse = self.all_verses[idx]
                 surah_name = self.surahs[verse['surah_id'] - 1]['surahNameArabicLong']
-                results.append(
-                    f"سورة {surah_name} - آية {verse['verse_num']}:\n{verse['text']}"
-                )
+                results.append(f"سورة {surah_name} - آية {verse['verse_num']}:\n{verse['text']}")
             return "\n\n".join(results)
         except Exception as e:
-            self.logger.error(f"Search failed: {e}")
+            self.logger.error(f"Search failed: {e}", exc_info=True)
             return "حدث خطأ أثناء البحث. جرب مرة أخرى."
 
     def _keyword_fallback_search(self, query, top_k=5):
@@ -128,11 +133,7 @@ class QuranSearchEngine:
         return "\n\n".join(matches[:top_k]) or "لا توجد نتائج مطابقة."
 
     def _load_fallback_surahs(self):
-        return [
-            ("الفاتحة", 1),
-            ("البقرة", 2),
-            ("آل عمران", 3)
-        ]
+        return [("الفاتحة", 1), ("البقرة", 2), ("آل عمران", 3)]
 
     def _load_fallback_verse(self):
         return "بسم الله الرحمن الرحيم\nالله لا إله إلا هو الحي القيوم"