Update tools/quran_search.py

tools/quran_search.py

@@ -1,134 +1,89 @@
 import logging
-import pandas as pd
+import requests
 from sentence_transformers import SentenceTransformer
 from sklearn.metrics.pairwise import cosine_similarity
 import numpy as np
-import requests
 
 class QuranSearchEngine:
     def __init__(self):
-        self.data_loaded = False
-        self.model = None
-        self.verse_embeddings = None
-        self.quran_df = None
-        self.surah_names = {}
-        self.base_api_url = "https://quranapi.pages.dev/api/verses"
+        self.api_url = "https://api.quran.com/api/v3/search"
+        self.model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2', device='cpu')
+        self.embedding_cache = {}
+        self.min_query_length = 2
         
-        # Initialize with progress tracking
-        self.load_data()
+        # Configure logging
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s'
+        )
 
-    def load_data(self):
-        if not self.data_loaded:
-            try:
-                # Step 1: Load Surah names
-                self._load_surah_names()
-                
-                # Step 2: Fetch verses in batches
-                all_verses = []
-                for surah_num in range(1, 115):  # All 114 Surahs
-                    verses = self._fetch_verses(surah_num)
-                    if verses:
-                        all_verses.extend(verses)
-                
-                # Step 3: Create DataFrame
-                self.quran_df = pd.DataFrame(all_verses)
-                
-                # Step 4: Initialize model
-                self.model = SentenceTransformer(
-                    'paraphrase-multilingual-MiniLM-L12-v2',
-                    device='cpu'
-                )
-                
-                # Step 5: Generate embeddings in chunks
-                texts = self.quran_df['text'].tolist()
-                self.verse_embeddings = np.concatenate([
-                    self.model.encode(texts[i:i+100])
-                    for i in range(0, len(texts), 100)
-                ])
-                
-                self.data_loaded = True
-                logging.info("Quran data loaded successfully")
-                
-            except Exception as e:
-                logging.error(f"Data loading failed: {str(e)}")
-                self._load_backup_data()
-
-    def _load_surah_names(self):
-        """Fetch surah names from API"""
-        try:
-            response = requests.get(f"{self.base_api_url}/surahs")
-            if response.status_code == 200:
-                surahs = response.json()
-                self.surah_names = {s['number']: s['name'] for s in surahs}
-        except Exception as e:
-            logging.warning(f"Couldn't fetch surah names: {str(e)}")
-            # Fallback to minimal names
-            self.surah_names = {i: f"سورة {i}" for i in range(1, 115)}
-
-    def _fetch_verses(self, surah_num):
-        """Fetch verses for a specific surah"""
+    def _fetch_verses(self, query, limit=5):
+        """Fetch verses from Quran API with error handling"""
         try:
             response = requests.get(
-                f"{self.base_api_url}/{surah_num}",
-                timeout=10
+                f"{self.api_url}?q={query}&size={limit}",
+                timeout=15,
+                headers={'Accept': 'application/json'}
             )
-            if response.status_code == 200:
-                verses_data = response.json()
-                return [{
-                    'surah': surah_num,
-                    'ayah': v['verse'],
-                    'text': v['text'],
-                    'surah_name': self.surah_names.get(surah_num, "")
-                } for v in verses_data]
-        except Exception as e:
-            logging.warning(f"Failed to fetch surah {surah_num}: {str(e)}")
+            response.raise_for_status()
+            return response.json().get('results', [])
+        except requests.exceptions.RequestException as e:
+            logging.error(f"API request failed: {str(e)}")
+            return []
+        except ValueError as e:
+            logging.error(f"Invalid API response: {str(e)}")
             return []
 
-    def _load_backup_data(self):
-        """Emergency fallback"""
-        backup = [
-            {"surah": 1, "ayah": 1, "text": "بسم الله الرحمن الرحيم", "surah_name": "الفاتحة"},
-            {"surah": 2, "ayah": 255, "text": "الله لا إله إلا هو الحي القيوم...", "surah_name": "البقرة"},
-            {"surah": 36, "ayah": 1, "text": "يس والقرآن الحكيم", "surah_name": "يس"}
-        ]
-        self.quran_df = pd.DataFrame(backup)
-        self.model = SentenceTransformer('paraphrase-multilingual-MiniLM-L12-v2')
-        self.verse_embeddings = self.model.encode(self.quran_df['text'].tolist())
-        self.data_loaded = True
-        logging.warning("Using backup data")
+    def _process_verse(self, verse, similarity):
+        """Standardize verse format"""
+        return {
+            'surah': verse.get('surah_name', ''),
+            'ayah': verse.get('verse_id', 0),
+            'text': verse.get('text', ''),
+            'similarity': f"{similarity:.2f}",
+            'surah_num': verse.get('surah_id', 0),
+            'ayah_num': verse.get('verse_id', 0)
+        }
 
     def search(self, query, top_k=5):
-        if not self.data_loaded:
+        """Main search method with validation and caching"""
+        # Validate input
+        query = str(query).strip()
+        if len(query) < self.min_query_length:
             return []
-            
+
         try:
-            # Clean and validate query
-            query = str(query).strip()
-            if len(query) < 2:
+            # 1. Get initial results from API
+            verses = self._fetch_verses(query, top_k)
+            if not verses:
                 return []
-                
-            # Encode query and calculate similarities
-            query_embedding = self.model.encode([query])
-            similarities = cosine_similarity(query_embedding, self.verse_embeddings)[0]
+
+            # 2. Prepare texts for embedding
+            texts = [v['text'] for v in verses]
             
-            # Get top results
-            top_indices = np.argsort(similarities)[-top_k:][::-1]
+            # 3. Get or create embeddings
+            if query in self.embedding_cache:
+                query_embedding = self.embedding_cache[query]
+            else:
+                query_embedding = self.model.encode([query])[0]
+                self.embedding_cache[query] = query_embedding
+
+            verse_embeddings = self.model.encode(texts)
             
-            # Format results
-            results = []
-            for idx in top_indices:
-                verse = self.quran_df.iloc[idx]
-                results.append({
-                    "surah": verse['surah_name'],
-                    "ayah": verse['ayah'],
-                    "text": verse['text'],
-                    "similarity": f"{similarities[idx]:.2f}",
-                    "surah_num": verse['surah'],
-                    "ayah_num": verse['ayah']
-                })
-                
-            return results
+            # 4. Calculate similarities
+            similarities = cosine_similarity(
+                [query_embedding],
+                verse_embeddings
+            )[0]
+
+            # 5. Combine and sort results
+            results = [
+                self._process_verse(verse, similarities[i])
+                for i, verse in enumerate(verses)
+            ]
             
+            return sorted(results, key=lambda x: float(x['similarity']), reverse=True)
+
         except Exception as e:
-            logging.error(f"Search error: {str(e)}")
+            logging.error(f"Search processing failed: {str(e)}")
             return []