Upload folder using huggingface_hub

db.py

@@ -77,43 +77,81 @@ class SanatanDatabase:
             metadatas=[metas[i] for i in indices],
         )
 
-    def fetch_first_match(
+    def fetch_first_match( 
         self, collection_name: str, metadata_where_clause: MetadataWhereClause = None
     ):
-        """This version is created to support the browse module"""
+        """This version is created to support the browse module with fallback regex matching"""
         logger.info(
             "getting first matching verses from [%s] | metadata_where_clause = %s",
             collection_name,
             metadata_where_clause,
         )
         collection = self.chroma_client.get_or_create_collection(name=collection_name)
-        data = collection.get(
-            include=["metadatas", "documents"],
-            where=(
-                metadata_where_clause.to_chroma_where()
-                if metadata_where_clause is not None
-                else None
-            ),
+
+        where_clause = (
+            metadata_where_clause.to_chroma_where()
+            if metadata_where_clause is not None
+            else None
         )
 
+        data = collection.get(include=["metadatas", "documents"], where=where_clause)
+
         if data["metadatas"]:
-            # find index of record with lowest _global_index
+            # ✅ normal path
             min_index = min(
                 range(len(data["metadatas"])),
                 key=lambda i: data["metadatas"][i].get("_global_index", float("inf")),
             )
-
-            # shrink data to keep same structure but only one record
-            data = {
+            return {
                 "ids": [data["ids"][min_index]],
                 "documents": [data["documents"][min_index]],
                 "metadatas": [data["metadatas"][min_index]],
             }
-        else:
-            logger.warning("No data found! - data=%s", data)
+
+        # ⚠️ fallback path
+        logger.warning("No data found using strict filter. Attempting regex fallback.")
+
+        if not metadata_where_clause or not metadata_where_clause.filters:
+            return chromadb.GetResult(ids=[], documents=[], metadatas=[])
+
+        # find filters with $eq string type
+        regex_filters = [
+            f for f in metadata_where_clause.filters
+            if f.metadata_search_operator == "$eq" and isinstance(f.metadata_value, str)
+        ]
+
+        if not regex_filters:
             return chromadb.GetResult(ids=[], documents=[], metadatas=[])
 
-        return data
+        # Pull all documents for manual regex scan
+        all_data = collection.get(include=["metadatas", "documents"])
+
+        matched_indices = []
+        for i, meta in enumerate(all_data["metadatas"]):
+            ok = True
+            for f in regex_filters:
+                field_val = str(meta.get(f.metadata_field, ""))
+                # case-insensitive substring search as fallback
+                if not re.search(re.escape(f.metadata_value), field_val, flags=re.IGNORECASE):
+                    ok = False
+                    break
+            if ok:
+                matched_indices.append(i)
+
+        if not matched_indices:
+            logger.warning("Regex fallback also found no matches.")
+            return chromadb.GetResult(ids=[], documents=[], metadatas=[])
+
+        # Pick lowest _global_index among matches
+        min_index = min(
+            matched_indices,
+            key=lambda i: all_data["metadatas"][i].get("_global_index", float("inf")),
+        )
+        return {
+            "ids": [all_data["ids"][min_index]],
+            "documents": [all_data["documents"][min_index]],
+            "metadatas": [all_data["metadatas"][min_index]],
+        }
 
     def search(
         self,

modules/config/divya_prabandham.py

@@ -78,23 +78,6 @@ divya_prabandham_config = {
                 for divya_desam_name in nalayiram_helper.get_standardized_divya_desam_names()
             ],
         },
-        {
-            "name": "title",
-            "label": "Pasuram Title",
-            "datatype": "str",
-            "description": (
-                "Exact title of a pasuram in one of the following formats:\n"
-                "1. '{prabandham_code} {decade}.{chapter}.{pasuram}' — use when the prabandham has decades.\n"
-                "2. '{prabandham_code} {chapter}.{pasuram}' — use when the prabandham does not have decades.\n\n"
-                "⚠️ Use this field ONLY when the user provides a specific prabandham and a relative verse number.\n"
-                "Examples of valid usage:\n"
-                "- User query: '3rd pasuram in the 8th Thiruvaimozhi of the 1st decade.'\n"
-                "  → Convert to: '{prabandham_code} 1.8.3' and pass as `title` filter.\n"
-                "- User query: '2nd pasuram of chapter 5 in [Prabandham with no decades].'\n"
-                "  → Convert to: '{prabandham_code} 5.2' and pass as `title` filter.\n"
-                "Do NOT use `title` for general queries or keyword searches — leave it empty in those cases."
-            ),
-        },
         {
             "name": "verse",
             "label": "Absolute Pasuram Number",
@@ -136,6 +119,38 @@ divya_prabandham_config = {
             ),
             "show_as_filter": True,
         },
+        {
+            "name": "title",
+            "label": "Pasuram Title",
+            "datatype": "str",
+            "description": (
+                "Exact title of a pasuram in one of the following formats:\n"
+                "1. '{prabandham_code} {decade}.{chapter}.{pasuram}' — use when the prabandham has decades.\n"
+                "2. '{prabandham_code} {chapter}.{pasuram}' — use when the prabandham does not have decades.\n\n"
+                "⚠️ Use this field ONLY when the user provides a specific prabandham and a relative verse number.\n"
+                "Examples of valid usage:\n"
+                "- User query: '3rd pasuram in the 8th Thiruvaimozhi of the 1st decade.'\n"
+                "  → Convert to: '{prabandham_code} 1.8.3' and pass as `title` filter.\n"
+                "- User query: '2nd pasuram of chapter 5 in [Prabandham with no decades].'\n"
+                "  → Convert to: '{prabandham_code} 5.2' and pass as `title` filter.\n"
+                "Do NOT use `title` for general queries or keyword searches — leave it empty in those cases."
+            ),
+            "show_as_filter": True,
+        },
+        {
+            "name": "pasuram_en",
+            "label": "Pasuram Lyrics in English",
+            "datatype": "str",
+            "description": "Pasuram lyrics in English",
+            "show_as_filter": True,
+        },
+        {
+            "name": "pasuram_ta",
+            "label": "Pasuram Lyrics in Tamil",
+            "datatype": "str",
+            "description": "Pasuram lyrics in Tamil",
+            "show_as_filter": True,
+        },
     ],
     "pdf_path": "./data/divya_prabandham.pdf",
     "source": "https://uveda.org",

modules/config/katakam.py

@@ -64,12 +64,14 @@ katakam_config = {
             "label": "Lyrics in sanskrit",
             "datatype": "str",
             "description": "The original sloka in sanskrit.",
+            "show_as_filter": True,
         },
         {
             "name": "transliteration",
             "label": "Transliteration in english",
             "datatype": "str",
             "description": "The original sloka transliterated in English.",
+            "show_as_filter": True,
         },
     ],
     "pdf_path": "./data/katakam.pdf",

modules/config/pancha_sooktham.py

@@ -31,12 +31,6 @@ pancha_sooktham_config = {
             "lov": lambda: get_pancha_sooktham_chapters(),
             "is_unique": True,
         },
-        {
-            "name": "sanskrit",
-            "label": "Lyrics in sanskrit",
-            "datatype": "str",
-            "description": "The original sloka in sanskrit.",
-        },
         {
             "name": "relative_verse_number",
             "label": "Relative Verse Number",
@@ -45,6 +39,13 @@ pancha_sooktham_config = {
             "show_as_filter": True,
             "is_unique": True,
         },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "The original sloka in sanskrit.",
+            "show_as_filter": True,
+        },
     ],
     "pdf_path": "./data/pancha_sooktham.pdf",
     "source": "https://stotranidhi.com/",

modules/config/shanthi_panchakam.py

@@ -32,12 +32,6 @@ shanthi_panchakam_config = {
             "lov": lambda: get_shanthi_panchakam_chapters(),
             "is_unique": True,
         },
-        {
-            "name": "sanskrit",
-            "label": "Lyrics in sanskrit",
-            "datatype": "str",
-            "description": "The original sloka in sanskrit.",
-        },
         {
             "name": "relative_verse_number",
             "label": "Relative Verse Number",
@@ -46,6 +40,13 @@ shanthi_panchakam_config = {
             "show_as_filter": True,
             "is_unique": True,
         },
+        {
+            "name": "sanskrit",
+            "label": "Lyrics in sanskrit",
+            "datatype": "str",
+            "description": "The original sloka in sanskrit.",
+            "show_as_filter": True,
+        },
     ],
     "pdf_path": "./data/shanthi_panchakam.pdf",
     "source": "https://vignanam.org/english/shanti-panchakam.html",

modules/config/sri_stavam.py

@@ -29,23 +29,26 @@ sri_stavam_config = {
             "show_as_filter": True,
             "is_unique": True,
         },
-        {
-            "name": "meaning_short",
-            "label": "Short meaning",
-            "datatype": "str",
-            "description": "A short meaning of the sanskrit verse in English.",
-        },
         {
             "name": "sanskrit",
             "label": "Lyrics in sanskrit",
             "datatype": "str",
             "description": "Verse in sanskrit",
+            "show_as_filter": True,
         },
         {
             "name": "transliteration",
             "label": "English Transliteration",
             "datatype": "str",
             "description": "Verse transliterated to English",
+            "show_as_filter": True,
+        },
+        {
+            "name": "meaning_short",
+            "label": "Short meaning",
+            "datatype": "str",
+            "description": "A short meaning of the sanskrit verse in English.",
+            "show_as_filter": True,
         },
     ],
     "pdf_path": "./data/sri_stavam.pdf",

modules/config/taitriya_aranyakam.py

@@ -83,12 +83,14 @@ taitriya_aranyakam_config = {
             "label": "Lyrics in sanskrit",
             "datatype": "str",
             "description": "The original sloka in sanskrit.",
+            "show_as_filter": True,
         },
         {
             "name": "transliteration",
             "label": "Transliteration in english",
             "datatype": "str",
             "description": "The original sloka transliterated in English.",
+            "show_as_filter": True,
         },
     ],
     "pdf_path": "./data/taitriya_aranyakam.pdf",

modules/config/taitriya_brahmanam.py

@@ -76,12 +76,14 @@ taitriya_brahmanam_config = {
             "label": "Lyrics in sanskrit",
             "datatype": "str",
             "description": "The original sloka in sanskrit.",
+            "show_as_filter": True,
         },
         {
             "name": "transliteration",
             "label": "Transliteration in english",
             "datatype": "str",
             "description": "The original sloka transliterated in English.",
+            "show_as_filter": True,
         },
     ],
     "pdf_path": "./data/taitriya_brahmanam.pdf",

modules/config/taitriya_samhitha.py

@@ -76,12 +76,14 @@ taitriya_samhitha_config = {
             "label": "Lyrics in sanskrit",
             "datatype": "str",
             "description": "The original sloka in sanskrit.",
+            "show_as_filter": True,
         },
         {
             "name": "transliteration",
             "label": "Transliteration in english",
             "datatype": "str",
             "description": "The original sloka transliterated in English.",
+            "show_as_filter": True,
         },
     ],
     "pdf_path": "./data/taitriya_samhitha.pdf",

modules/config/taitriya_upanishad.py

@@ -32,12 +32,6 @@ taitriya_upanishad_config = {
                     "lov": lambda: get_taitriya_upanishad_chapters(),
                     "is_unique": True,
                 },
-                {
-                    "name": "sanskrit",
-                    "label": "Lyrics in sanskrit",
-                    "datatype": "str",
-                    "description": "The original sloka in sanskrit.",
-                },
                 {
                     "name": "relative_verse_number",
                     "label": "Relative Verse Number",
@@ -46,6 +40,13 @@ taitriya_upanishad_config = {
                     "show_as_filter": True,
                     "is_unique": True,
                 },
+                {
+                    "name": "sanskrit",
+                    "label": "Lyrics in sanskrit",
+                    "datatype": "str",
+                    "description": "The original sloka in sanskrit.",
+                    "show_as_filter": True,
+                },
             ],
             "pdf_path": "./data/taitriya_upanishad.pdf",
             "source": "https://stotranidhi.com/",

modules/config/vishnu_sahasranamam.py

@@ -35,36 +35,33 @@ vishnu_sahasranamam_config = {
             "is_unique": True,
         },
         {
-            "name": "page_number",
+            "name": "verse",
+            "label": "Verse Number",
             "datatype": "int",
-            "label": "Page Number",
-            "description": "Page number from the source",
+            "description": "The verse number of the sloka.",
+            "show_as_filter": True,
+            "is_unique": True,
         },
         {
             "name": "sanskrit",
             "label": "Lyrics in sanskrit",
             "datatype": "str",
             "description": "The original sloka in sanskrit.",
+            "show_as_filter": True,
         },
         {
             "name": "translation",
             "label": "English Translation",
             "datatype": "str",
             "description": "The english translation.",
+            "show_as_filter": True,
         },
         {
             "name": "transliteration",
             "label": "English Transliteration",
             "datatype": "str",
             "description": "The english transliteration.",
-        },
-        {
-            "name": "verse",
-            "label": "Verse Number",
-            "datatype": "int",
-            "description": "The verse number of the sloka.",
             "show_as_filter": True,
-            "is_unique": True,
         },
     ],
     "pdf_path": "./data/vishnu_sahasranamam.pdf",

server.py

@@ -243,7 +243,7 @@ async def get_scripture(req: ScriptureRequest):
     # add unit index & total units (so Flutter can paginate)
     canonical_doc["total"] = SanatanDatabase().count(config["collection_name"])
 
-    print("canonical_doc = ", canonical_doc)
+    # print("canonical_doc = ", canonical_doc)
     return canonical_doc
 
 
@@ -295,6 +295,7 @@ async def search_scripture(
     - `n_results`: number of random results to return
     """
     try:
+        logger.info("search_scripture: searching for %s with filters %s", scripture_name, filter_obj)
         db = SanatanDatabase()
         config = next(
             (s for s in SanatanConfig().scriptures if s["name"] == scripture_name), None
@@ -305,14 +306,14 @@ async def search_scripture(
             metadata_where_clause=filter_obj,
         )
 
-        print("results = ", results)
+        # print("results = ", results)
         # Flatten + canonicalize results
         formatted_results = []
         for i in range(len(results["metadatas"])):
             id = results["ids"][i]
             metadata_doc = results["metadatas"][i]
             metadata_doc["id"] = id
-            print("metadata_doc = ", metadata_doc)
+            # print("metadata_doc = ", metadata_doc)
             document_text = (
                 results["documents"][i] if results.get("documents") else None
             )
@@ -322,7 +323,7 @@ async def search_scripture(
             )
             formatted_results.append(canonical_doc)
 
-        print("formatted_results = ", formatted_results)
+        # print("formatted_results = ", formatted_results)
         return {"results": formatted_results}
 
     except Exception as e: