Upload folder using huggingface_hub

config.py

@@ -15,11 +15,17 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "file",
+                    "label": "File Name",
                     "datatype": "str",
-                    "desc": "name of the file from which the information was extracted",
+                    "description": "name of the file from which the information was extracted",
+                },
+                {
+                    "name": "page",
+                    "datatype": "int",
+                    "label": "Page Number",
+                    "description": "Page number from the source",
+                    "show_as_filter" : True
                 },
-                {"name": "num_chars", "datatype": "str"},
-                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/vishnu_puranam.pdf",
             "source": "https://dn720005.ca.archive.org/0/items/vishnu-purana-sanskrit-english-ocr/VISHNU-PURANA-Sanskrit-English-OCR.pdf",
@@ -49,11 +55,17 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "file",
+                    "label": "File Name",
                     "datatype": "str",
-                    "desc": "name of the file from which the information was extracted",
+                    "description": "name of the file from which the information was extracted",
+                },
+                {
+                    "name": "page",
+                    "datatype": "int",
+                    "label": "Page Number",
+                    "description": "Page number from the source",
+                    "show_as_filter" : True
                 },
-                {"name": "num_chars", "datatype": "str"},
-                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/shukla-yajur-veda.pdf",
             "source": "https://www.thearyasamaj.org/uploads/book/2014/04/R1sSjG_eLb_sub_406_yajurveda.pdf",
@@ -80,11 +92,17 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "file",
+                    "label": "File Name",
                     "datatype": "str",
-                    "desc": "name of the file from which the information was extracted",
+                    "description": "name of the file from which the information was extracted",
+                },
+                {
+                    "name": "page",
+                    "datatype": "int",
+                    "label": "Page Number",
+                    "description": "Page number from the source",
+                    "show_as_filter" : True
                 },
-                {"name": "num_chars", "datatype": "str"},
-                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/bhagavat_gita.pdf",
             "source": "https://dn790006.ca.archive.org/0/items/in.gov.ignca.279/279_text.pdf",
@@ -117,11 +135,17 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "file",
+                    "label": "File Name",
                     "datatype": "str",
-                    "desc": "name of the file from which the information was extracted",
+                    "description": "name of the file from which the information was extracted",
+                },
+                {
+                    "name": "page",
+                    "datatype": "int",
+                    "label": "Page Number",
+                    "description": "Page number from the source",
+                    "show_as_filter" : True
                 },
-                {"name": "num_chars", "datatype": "str"},
-                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/valmiki_ramayanam.pdf",
             "source": "https://ia800509.us.archive.org/28/items/valmiki-ramayana-gita-press-english/Valmiki%20Ramayana%20Gita%20Press%20English.pdf",
@@ -163,27 +187,45 @@ class SanatanConfig:
                 "relative_path": lambda doc: f"Vishnu Sahasranamam-{doc.get("chapter","")}-{doc.get("verse","")}",
             },
             "metadata_fields": [
-                {"name": "chapter", "datatype": "str"},
-                {"name": "page_number", "datatype": "int"},
+                {
+                    "name": "chapter",
+                    "datatype": "str",
+                    "label": "Chapter Name",
+                    "description": "Name of the Chapter",
+                },
+                {
+                    "name": "page_number",
+                    "datatype": "int",
+                    "label": "Page Number",
+                    "description": "Page number from the source",
+                },
                 {
                     "name": "sanskrit",
+                    "label": "Lyrics in sanskrit",
                     "datatype": "str",
-                    "desc": "The original sloka in sanskrit.",
+                    "description": "The original sloka in sanskrit.",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "translation",
+                    "label": "English Translation",
                     "datatype": "str",
-                    "desc": "The english translation.",
+                    "description": "The english translation.",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "transliteration",
+                    "label": "English Transliteration",
                     "datatype": "str",
-                    "desc": "The english transliteration.",
+                    "description": "The english transliteration.",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "verse",
+                    "labek": "Verse Number",
                     "datatype": "int",
-                    "desc": "The verse number of the sloka.",
+                    "description": "The verse number of the sloka.",
+                    "show_as_filter" : True
                 },
             ],
             "pdf_path": "./data/vishnu_sahasranamam.pdf",
@@ -231,29 +273,35 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "prabandham_code",
+                    "label": "Prabandham Code",
                     "datatype": "str",
                     "description": "contains the short prabandham_code. e.g. `TPL` for `Thiruppallandu`",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "prabandham_name",
+                    "label": "Prabandham Name",
                     "datatype": "str",
                     "description": "contains the prabandham name. e.g. `Thiruppallandu`",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "azhwar_name",
+                    "label": "Azhwar Name",
                     "datatype": "str",
                     "description": "contains the azhwar name. e.g. `Thirumangai Azhwar`",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "divya_desams",
+                    "label": "Divya Desams",
                     "datatype": "str",
                     "description": "comma separated list of divya desams. e.g. Thiruneermalai,Thiruvallikkeni.",
+                    "show_as_filter" : True
                 },
-                # {"name": "html_url", "datatype": "str", "description" : "Reference link for the source"},
-                # {"name": "pasuram_en", "datatype": "str", "description" : "Transliteration of pasuram in english"},
-                # {"name": "pasuram_ta", "datatype": "str", "description" : "Pasuram lyrics in tamil"},
                 {
                     "name": "title",
+                    "label": "Pasuram Title",
                     "datatype": "str",
                     "description": (
                         "Exact title of a pasuram in one of the following formats:\n"
@@ -267,9 +315,11 @@ class SanatanConfig:
                         "  → Convert to: '{prabandham_code} 5.2' and pass as `title` filter.\n"
                         "Do NOT use `title` for general queries or keyword searches — leave it empty in those cases."
                     ),
+                    "show_as_filter" : True
                 },
                 {
                     "name": "verse",
+                    "label": "Absolute Pasuram Number",
                     "datatype": "int",
                     "is_unique": True,
                     "description": (
@@ -277,30 +327,36 @@ class SanatanConfig:
                         # "Use it only when a specific prabandham name is NOT mentioned in the user query."
                         "For e.g. 'Give me pasuram 1176'"
                     ),
+                    "show_as_filter" : True
                 },
-                # {"name": "wbw_ta", "datatype": "str", "description" : "Word by word meaning in tamil."},
                 {
                     "name": "decade",
+                    "label": "Decade Number (Pathu)",
                     "datatype": "int",
                     "description": (
                         "The decade (or `pathu` in Tamil) that this pasuram belongs to. decade is -1 when there is no associated decade."
                     ),
+                    "show_as_filter" : True
                 },
                 {
                     "name": "chapter",
+                    "label": "Chapter Number",
                     "datatype": "int",
                     "description": (
                         "chapter number of this pasuram. is -1 when there is no associated chapter number"
                     ),
+                    "show_as_filter" : True
                 },
                 {
                     "name": "position_in_chapter",
+                    "label": "Relative Pasuram Number",
                     "datatype": "int",
                     "description": (
                         "Relative verse number or pasuram number within a chapter."
                         "Use it only when a specific prabandham name is mentioned in the user query."
                         "For e.g. 'Give me the 5th pasuram from Thirupavai'"
                     ),
+                    "show_as_filter" : True
                 },
             ],
             "pdf_path": "./data/divya_prabandham.pdf",
@@ -339,11 +395,17 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "file",
+                    "label": "File Name",
                     "datatype": "str",
-                    "desc": "name of the file from which the information was extracted",
+                    "description": "name of the file from which the information was extracted",
+                },
+                {
+                    "name": "page",
+                    "datatype": "int",
+                    "label": "Page Number",
+                    "description": "Page number from the source",
+                    "show_as_filter" : True
                 },
-                {"name": "num_chars", "datatype": "str"},
-                {"name": "page", "datatype": "int"},
             ],
             "pdf_path": "./data/bhagavata_purana.pdf",
             "source": "https://dn790003.ca.archive.org/0/items/bhagavatapuranagitapress_201907/Bhagavata%20Purana%20-%20Gita%20Press_text.pdf",
@@ -373,21 +435,38 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "kandam",
+                    "label": "Kandam",
                     "datatype": "str",
                     "description": "The name of the Kandam or the chapter.",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "padalam_en",
+                    "label": "Chapter Name",
                     "datatype": "str",
                     "description": "The name of the Padalam (Episode) in English.",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "padalam_ta",
+                    "label": "Padalam name in Tamil",
                     "datatype": "str",
                     "description": "The name of the Padalam (Episode) in Tamil.",
                 },
-                {"name": "page", "datatype": "int"},
-                {"name": "verse_number", "datatype": "int"},
+                {
+                    "name": "page",
+                    "datatype": "int",
+                    "label": "Page Number",
+                    "description": "Page number from the source",
+                    "show_as_filter" : True
+                },
+                {
+                    "name": "verse_number",
+                    "datatype": "int",
+                    "label": "Verse Number",
+                    "description": "Verse Number",
+                    "show_as_filter" : True
+                },
             ],
             "pdf_path": "./data/kamba_ramayanam.pdf",
             "source": "https://www.hindupedia.com/images/1/13/Kamba_Ramayanam_I.pdf",
@@ -426,11 +505,14 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "chunk_index",
+                    "label" : "Page Index",
                     "datatype": "int",
                     "description": "The index of the chunk",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "filename",
+                    "label" : "File name from the source",
                     "datatype": "str",
                     "description": "The name of the file.",
                 },
@@ -473,13 +555,17 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "sloka_number",
+                    "label" : "Slokam Number",
                     "datatype": "int",
                     "description": "The index of the sloka or verse",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "meaning_short",
+                    "label" : "Short meaning",
                     "datatype": "str",
                     "description": "A short meaning of the sanskrit verse in English.",
+                    "show_as_filter" : True
                 },
             ],
             "pdf_path": "./data/chathusloki.pdf",
@@ -512,23 +598,30 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "sloka_number",
+                    "label" : "Slokam Number",
                     "datatype": "int",
                     "description": "The index of the sloka or verse",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "meaning_short",
+                    "label" : "Short meaning",
                     "datatype": "str",
                     "description": "A short meaning of the sanskrit verse in English.",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "sanskrit",
+                    "label" : "Lyrics in sanskrit",
                     "datatype": "str",
                     "description": "Verse in sanskrit",
                 },
                 {
                     "name": "transliteration",
+                    "label" : "English Transliteration",
                     "datatype": "str",
                     "description": "Verse transliterated to English",
+                    "show_as_filter" : True
                 },
             ],
             "pdf_path": "./data/sri_stavam.pdf",
@@ -560,28 +653,37 @@ class SanatanConfig:
             "metadata_fields": [
                 {
                     "name": "video_id",
+                    "label" : "Video ID",
                     "datatype": "str",
                     "description": "The video id as in YouTube",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "video_title",
+                    "label" : "Video Title",
                     "datatype": "str",
                     "description": "The title of the video as in YouTube",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "description",
+                    "label" : "Video Description",
                     "datatype": "str",
                     "description": "Description as in YouTube",
+                    "show_as_filter" : True
                 },
                 {
                     "name": "channel_url",
+                    "label" : "Channel URL",
                     "datatype": "str",
                     "description": "URL of the YouTube Channel",
                 },
                 {
                     "name": "channel_title",
+                    "label" : "Channel Title",
                     "datatype": "str",
                     "description": "Title of the YouTube Channel",
+                    "show_as_filter" : True
                 },
             ],
             "pdf_path": "./data/none.pdf",

db.py

@@ -151,7 +151,7 @@ class SanatanDatabase:
             result = {"document": documents[0]}
             if metadatas:
                 result.update(metadatas[0])
-            print("raw data = ", result)
+            # print("raw data = ", result)
             return result
         else:
             print("No data available")

server.py

@@ -11,6 +11,7 @@ from pydantic import BaseModel
 from chat_utils import chat
 from config import SanatanConfig
 from db import SanatanDatabase
+from metadata import MetadataWhereClause
 from modules.quiz.answer_validator import validate_answer
 from modules.quiz.models import Question
 from modules.quiz.quiz_helper import generate_question
@@ -245,10 +246,54 @@ async def get_scripture_configs():
                 "name": s["name"],  # e.g. "bhagavad_gita"
                 "title": s["title"],  # e.g. "Bhagavad Gita"
                 "unit": s["unit"],  # e.g. "verse" or "page"
+                "unit_field": s.get("unit_field", s.get("unit")),
                 "total": num_units,
                 "enabled": "field_mapping" in s,
-                "source" : s.get("source",""),
+                "source": s.get("source", ""),
                 "credits": s.get("credits", f"{s.get("source","")}"),
+                "metadata_fields": s.get("metadata_fields", []),
             }
         )
     return {"scriptures": sorted(scriptures, key=lambda s: s["title"])}
+
+
+@router.post("/scripture/{scripture_name}/search")
+async def search_scripture(
+    scripture_name: str,
+    filter_obj: Optional[MetadataWhereClause] = None,
+    n_results: int = 1,
+):
+    """
+    Search scripture collection with optional filters.
+    - `scripture_name`: Name of the collection
+    - `filter_obj`: MetadataWhereClause (filters, groups, operator)
+    - `n_results`: number of random results to return
+    """
+    try:
+        db = SanatanDatabase()
+        results = db.fetch_random_data(
+            collection_name=scripture_name,
+            metadata_where_clause=filter_obj,
+            n_results=n_results,
+        )
+
+        print("results = ", results)
+        # Flatten + canonicalize results
+        formatted_results = []
+        for i in range(len(results["metadatas"])):
+            metadata_doc = results["metadatas"][i]
+            document_text = (
+                results["documents"][i] if results.get("documents") else None
+            )
+
+            canonical_doc = SanatanConfig().canonicalize_document(
+                scripture_name, document_text, metadata_doc
+            )
+            formatted_results.append(canonical_doc)
+
+        # print("formatted_results = ", formatted_results)
+        return {"results": formatted_results}
+
+    except Exception as e:
+        print("Error while searching ", e)
+        return {"error": str(e)}