Upload folder using huggingface_hub

copy_chromadb.py

@@ -242,7 +242,33 @@ db_config = {
     "source_db_path": "../stotra_patam_ai/chromadb_store",
     "source_collection_name": "ranganatha_stotram",
     "destination_collection_name": "ranganatha_stotram"
-  }  
+  },
+    "devaraja_mangalam": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "devaraja_mangalam",
+    "destination_collection_name": "devaraja_mangalam"
+  },
+  "dhati_panchakam": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "dhati_panchakam",
+    "destination_collection_name": "dhati_panchakam"
+  },
+  "mukthaka_mangalam": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "mukthaka_mangalam",
+    "destination_collection_name": "mukthaka_mangalam"
+  },
+  "venkateswara_stotram": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "venkateswara_stotram",
+    "destination_collection_name": "venkateswara_stotram"
+  },
+  "yathiraja_vimsathi": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "yathiraja_vimsathi",
+    "destination_collection_name": "yathiraja_vimsathi"
+  }
+
 }
 
 parser = argparse.ArgumentParser(description="My app with database parameter")

modules/config/__init__.py

@@ -9,6 +9,8 @@ from modules.config.brahmavaivarthapurana import brahmavaivarthapurana_config
 from modules.config.chathusloki import chathusloki_config
 from modules.config.desika_prabandham import desika_prabandham_config
 from modules.config.devaraja_ashtakam import devaraja_ashtakam_config
+from modules.config.devaraja_mangalam import devaraja_mangalam_config
+from modules.config.dhati_panchakam import dhati_panchakam_config
 from modules.config.divya_prabandham import divya_prabandham_config
 from modules.config.divya_prabandham_taniyans import divya_prabandham_taniyans_config
 from modules.config.garudapuranam import garudapuranam_config
@@ -20,6 +22,7 @@ from modules.config.kurmapuranam import kurmapuranam_config
 from modules.config.lingapuranam import lingapuranam_config
 from modules.config.markandeypuranam import markandeypuranam_config
 from modules.config.matsyapuranam import matsyapuranam_config
+from modules.config.mukthaka_mangalam import mukthaka_mangalam_config
 from modules.config.naradapuranam import naradapuranam_config
 from modules.config.narayaneeyam import narayaneeyam_config
 from modules.config.padmapuranam import padmapuranam_config
@@ -42,8 +45,10 @@ from modules.config.vaayupuraanam import vaayupuraanam_config
 from modules.config.vaazhi_thirunaamams import vaazhi_thirunaamams_config
 from modules.config.valmiki_ramayanam import valmiki_ramayanam_config
 from modules.config.varahapuranam import varahapuranam_config
+from modules.config.venkateswara_stotram import venkateswara_stotram_config
 from modules.config.vishnu_puranam import vishnupuranam_config
 from modules.config.vishnu_sahasranamam import vishnu_sahasranamam_config
+from modules.config.yathiraja_vimsathi import yathiraja_vimsathi_config
 from modules.config.yt_metadata import yt_metadata_config
 from modules.config.geethartha_sangraha import geethartha_sangraha_config
 from modules.config.mukunda_mala import mukunda_mala_config
@@ -104,4 +109,9 @@ scripture_configurations = [
     narasimha_ashtakam_config,
     panchayudha_stotram_config,
     ranganatha_stotram_config,
+    devaraja_mangalam_config,
+    mukthaka_mangalam_config,
+    venkateswara_stotram_config,
+    yathiraja_vimsathi_config,
+    dhati_panchakam_config,
 ]

modules/config/devaraja_mangalam.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+devaraja_mangalam_config = {
+    "name": "devaraja_mangalam",
+    "title": "Devaraja Mangalam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/devaraja_mangalam",
+    "collection_name": "devaraja_mangalam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Sri Manavala Mamunigal",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["SLOKAM"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/devaraja_mangalam.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "sanskritdocuments.org",
+                "url": "https://sanskritdocuments.org/doc_vishhnu/devarAjamangalam.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/dhati_panchakam.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+dhati_panchakam_config = {
+    "name": "dhati_panchakam",
+    "title": "Dhati Panchakam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/dhati_panchakam",
+    "collection_name": "dhati_panchakam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Shri Kooraththazhvan",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["SLOKAM", "THANIYAN"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/dhati_panchakam.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "vignanam.org",
+                "url": "https://vignanam.org/devanagari/dhati-panchakam.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/mukthaka_mangalam.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+mukthaka_mangalam_config = {
+    "name": "mukthaka_mangalam",
+    "title": "Mukthaka Mangalam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/mukthaka_mangalam",
+    "collection_name": "mukthaka_mangalam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Sri Manavala Mamunigal",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["SLOKAM", "THANIYAN"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/mukthaka_mangalam.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "stotranidhi.com",
+                "url": "https://stotranidhi.com/hi/mukthaka-mangalam-sri-manavala-mamunigal-in-sanskrit/",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/venkateswara_stotram.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+venkateswara_stotram_config = {
+    "name": "venkateswara_stotram",
+    "title": "Venkateswara Stotram",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/venkateswara_stotram",
+    "collection_name": "venkateswara_stotram",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Shri Prativadi Bayankara Annan Swami",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["SLOKAM"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/venkateswara_stotram.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "sanskritdocuments.org",
+                "url": "https://sanskritdocuments.org/doc_vishhnu/venkateshvara.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/yathiraja_vimsathi.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+yathiraja_vimsathi_config = {
+    "name": "yathiraja_vimsathi",
+    "title": "Yathiraja Vimsathi",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/yathiraja_vimsathi",
+    "collection_name": "yathiraja_vimsathi",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Sri Manavala Mamunigal",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["DYANA SLOKA", "SLOKAM"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/yathiraja_vimsathi.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "sanskritdocuments.org",
+                "url": "https://sanskritdocuments.org/doc_deities_misc/yatirAjaviMshati.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}