Upload folder using huggingface_hub

db.py

@@ -188,6 +188,77 @@ class SanatanDatabase:
             "metadatas": [all_data["metadatas"][min_index]],
         }
 
+    def count_where(
+        self,
+        collection_name: str,
+        metadata_where_clause: MetadataWhereClause = None,
+    ) -> int:
+        """
+        Count the number of matching verses in the collection without fetching documents.
+        Uses the same filtering and fallback logic as fetch_all_matches.
+        """
+
+        def normalize_for_match(s: str) -> str:
+            s = unicodedata.normalize("NFD", s)
+            s = "".join(ch for ch in s if not unicodedata.combining(ch))
+            return s
+
+        logger.info(
+            "count_where: counting matches in [%s] | filters=%s",
+            collection_name,
+            metadata_where_clause,
+        )
+
+        collection = self.chroma_client.get_or_create_collection(name=collection_name)
+        where_clause = (
+            metadata_where_clause.to_chroma_where() if metadata_where_clause else None
+        )
+
+        # If conversion returns an empty dict, treat as None
+        if isinstance(where_clause, dict) and not where_clause:
+            where_clause = None
+
+        # Strict filter first
+        data = collection.get(include=["metadatas"], where=where_clause)
+
+        if not data["metadatas"]:
+            # fallback regex
+            logger.warning("count_where: No matches found with strict filter. Trying regex fallback.")
+
+            if not metadata_where_clause or not metadata_where_clause.filters:
+                return 0
+
+            regex_filters = [
+                f
+                for f in metadata_where_clause.filters
+                if f.metadata_search_operator == "$eq"
+                and isinstance(f.metadata_value, str)
+            ]
+
+            if regex_filters:
+                all_data = collection.get(include=["metadatas"])
+                matched_count = 0
+                for meta in all_data["metadatas"]:
+                    ok = True
+                    for f in regex_filters:
+                        field_val = str(meta.get(f.metadata_field, ""))
+                        norm_val = normalize_for_match(field_val)
+                        norm_query = normalize_for_match(f.metadata_value)
+
+                        if not re.search(
+                            re.escape(norm_query), norm_val, flags=re.IGNORECASE
+                        ):
+                            ok = False
+                            break
+                    if ok:
+                        matched_count += 1
+                return matched_count
+            else:
+                return 0
+
+        # Direct count
+        return len(data["metadatas"])
+  
     def fetch_all_matches(
         self,
         collection_name: str,

server.py

@@ -443,22 +443,23 @@ async def search_scripture_find_all_matches(
     - `filter_obj`: MetadataWhereClause (filters, groups, operator)
     - `page`: 1-based page number
     - `page_size`: Number of results per page
-    - `has_audio` : optional. can take values any|none|recitation|virutham|upanyasam
+    - `has_audio`: optional. can take values any|none|recitation|virutham|upanyasam
     """
     filter_obj = req.filter_obj
-    page = req.page
-    page_size = req.page_size
+    page = req.page or 1
+    page_size = req.page_size or 20
     has_audio = req.has_audio
-    try:
-        logger.info(
-            "search_scripture_find_all_matches: searching for %s with filters %s | page=%s, page_size=%s, has_audio=%s",
-            scripture_name,
-            filter_obj,
-            page,
-            page_size,
-            has_audio,
-        )
 
+    logger.info(
+        "search_scripture_find_all_matches: searching for %s | filters=%s | page=%s | page_size=%s | has_audio=%s",
+        scripture_name,
+        filter_obj,
+        page,
+        page_size,
+        has_audio,
+    )
+
+    try:
         db = SanatanDatabase()
         config = next(
             (s for s in SanatanConfig().scriptures if s["name"] == scripture_name),
@@ -467,16 +468,17 @@ async def search_scripture_find_all_matches(
         if not config:
             return {"error": f"Scripture '{scripture_name}' not found"}
 
-        # 1️⃣ Fetch all matching metadata WITHOUT pagination yet
+        # 1️⃣ Decide how much to fetch
+        fetch_all = has_audio is not None
         results = db.fetch_all_matches(
             collection_name=config["collection_name"],
             metadata_where_clause=filter_obj,
-            page=None,  # Fetch all to apply audio filter
-            page_size=None,
+            page=None if fetch_all else page,
+            page_size=None if fetch_all else page_size,
         )
 
         formatted_results = []
-        all_indices = []  # Keep track of all _global_index
+        all_indices = []
         for i in range(len(results["metadatas"])):
             doc_id = results["ids"][i]
             metadata_doc = results["metadatas"][i]
@@ -491,10 +493,10 @@ async def search_scripture_find_all_matches(
             formatted_results.append(canonical_doc)
             all_indices.append(canonical_doc["_global_index"])
 
-        # 2️⃣ Apply has_audio filter
+        # 2️⃣ Apply audio filter only if requested
         if has_audio:
             if has_audio == AudioType.none:
-                # Fetch all indices that have any audio type
+                # Remove anything that *has* any audio
                 all_audio_indices = set()
                 for atype in [
                     AudioType.recitation,
@@ -505,15 +507,14 @@ async def search_scripture_find_all_matches(
                     indices = await svc_get_indices_with_audio(scripture_name, atype)
                     all_audio_indices.update(indices)
 
-                # Keep only indices that are NOT in all_audio_indices
                 formatted_results = [
                     r
                     for r in formatted_results
                     if r["_global_index"] not in all_audio_indices
                 ]
             else:
+                # Filter for specific or 'any'
                 if has_audio == AudioType.any:
-                    # Combine indices for all audio types
                     audio_indices = set()
                     for atype in [
                         AudioType.recitation,
@@ -530,16 +531,26 @@ async def search_scripture_find_all_matches(
                         await svc_get_indices_with_audio(scripture_name, has_audio)
                     )
 
-                # Keep only indices that match
                 formatted_results = [
                     r for r in formatted_results if r["_global_index"] in audio_indices
                 ]
 
-        # 3️⃣ Apply pagination on filtered results
-        total_matches = len(formatted_results)
-        start_idx = (page - 1) * page_size
-        end_idx = start_idx + page_size
-        paginated_results = formatted_results[start_idx:end_idx]
+        # 3️⃣ Paginate *after* filtering if needed
+        total_matches = None
+        if fetch_all:
+            total_matches = len(formatted_results)
+        else:
+            total_matches = db.count_where(
+                collection_name=config["collection_name"],
+                metadata_where_clause=filter_obj,
+            )
+
+        if fetch_all:
+            start_idx = (page - 1) * page_size
+            end_idx = start_idx + page_size
+            paginated_results = formatted_results[start_idx:end_idx]
+        else:
+            paginated_results = formatted_results
 
         return {
             "results": paginated_results,