Upload folder using huggingface_hub

copy_chromadb.py

@@ -212,8 +212,37 @@ db_config = {
     "source_db_path": "../thiruvaimozhi_nootrandhadhi_ai/chromadb_store",
     "source_collection_name": "thiruvaimozhi_nootrandhadhi",
     "destination_collection_name": "thiruvaimozhi_nootrandhadhi"
-  }
-
+  },
+  "devaraja_ashtakam": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "devaraja_ashtakam",
+    "destination_collection_name": "devaraja_ashtakam"
+  },
+  "geethartha_sangraha": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "geethartha_sangraha",
+    "destination_collection_name": "geethartha_sangraha"
+  },
+  "mukunda_mala": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "mukunda_mala",
+    "destination_collection_name": "mukunda_mala"
+  },
+  "narasimha_ashtakam": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "narasimha_ashtakam",
+    "destination_collection_name": "narasimha_ashtakam"
+  },
+  "panchayudha_stotram": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "panchayudha_stotram",
+    "destination_collection_name": "panchayudha_stotram"
+  },
+  "ranganatha_stotram": {
+    "source_db_path": "../stotra_patam_ai/chromadb_store",
+    "source_collection_name": "ranganatha_stotram",
+    "destination_collection_name": "ranganatha_stotram"
+  }  
 }
 
 parser = argparse.ArgumentParser(description="My app with database parameter")

modules/config/__init__.py

@@ -8,6 +8,7 @@ from modules.config.brahmapuranam import brahmapuranam_config
 from modules.config.brahmavaivarthapurana import brahmavaivarthapurana_config
 from modules.config.chathusloki import chathusloki_config
 from modules.config.desika_prabandham import desika_prabandham_config
+from modules.config.devaraja_ashtakam import devaraja_ashtakam_config
 from modules.config.divya_prabandham import divya_prabandham_config
 from modules.config.divya_prabandham_taniyans import divya_prabandham_taniyans_config
 from modules.config.garudapuranam import garudapuranam_config
@@ -44,7 +45,11 @@ from modules.config.varahapuranam import varahapuranam_config
 from modules.config.vishnu_puranam import vishnupuranam_config
 from modules.config.vishnu_sahasranamam import vishnu_sahasranamam_config
 from modules.config.yt_metadata import yt_metadata_config
-
+from modules.config.geethartha_sangraha import geethartha_sangraha_config
+from modules.config.mukunda_mala import mukunda_mala_config
+from modules.config.narasimha_ashtakam import narasimha_ashtakam_config
+from modules.config.panchayudha_stotram import panchayudha_stotram_config
+from modules.config.ranganatha_stotram import ranganatha_stotram_config
 
 scripture_configurations = [
     bhagavat_gita_config,
@@ -92,5 +97,11 @@ scripture_configurations = [
     varahapuranam_config,    
     vaazhi_thirunaamams_config,
     upadesa_rathnamalai_config,
-    thiruvaimozhi_nootrandhadhi_config
+    thiruvaimozhi_nootrandhadhi_config,
+    devaraja_ashtakam_config,
+    geethartha_sangraha_config,
+    mukunda_mala_config,
+    narasimha_ashtakam_config,
+    panchayudha_stotram_config,
+    ranganatha_stotram_config,
 ]

modules/config/devaraja_ashtakam.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+devaraja_ashtakam_config = {
+    "name": "devaraja_ashtakam",
+    "title": "Devaraja Ashtakam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/devaraja_ashtakam",
+    "collection_name": "devaraja_ashtakam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Thirukkacchi Nambi",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["DYANA SLOKA", "SLOKAM"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/devaraja_ashtakam.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "sanskritdocuments.org",
+                "url": "https://sanskritdocuments.org/doc_vishhnu/devarAjAShTakam.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/geethartha_sangraha.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+geethartha_sangraha_config = {
+    "name": "geethartha_sangraha",
+    "title": "Geethartha Sangraha",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/geethartha_sangraha",
+    "collection_name": "geethartha_sangraha",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Shri Yamunacharya",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["SLOKAM"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/geethartha_sangraha.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "sanskritdocuments.org",
+                "url": "https://sanskritdocuments.org/doc_giitaa/gitaarthasangraha.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/mukunda_mala.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+mukunda_mala_config = {
+    "name": "mukunda_mala",
+    "title": "Mukunda Mala",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/mukunda_mala",
+    "collection_name": "mukunda_mala",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Shri Kulashekhara Azhwar",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["DYANA SLOKA", "SLOKAM"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/mukunda_mala.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "sanskritdocuments.org",
+                "url": "https://sanskritdocuments.org/doc_vishhnu/mukundamAlA.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/narasimha_ashtakam.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+narasimha_ashtakam_config = {
+    "name": "narasimha_ashtakam",
+    "title": "Narasimha Ashtakam",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/narasimha_ashtakam",
+    "collection_name": "narasimha_ashtakam",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Swamy Vadhikesari Azhagiyamanavala Jeeyar",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["PHALA SLOKAM", "SLOKAM", "THANIYAN"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/narasimha_ashtakam.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "sanskritdocuments.org",
+                "url": "https://sanskritdocuments.org/doc_vishhnu/nrisimhAShTakam.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/panchayudha_stotram.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+panchayudha_stotram_config = {
+    "name": "panchayudha_stotram",
+    "title": "Panchayudha Stotram",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/panchayudha_stotram",
+    "collection_name": "panchayudha_stotram",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["SLOKAM"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/panchayudha_stotram.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "sanskritdocuments.org",
+                "url": "https://sanskritdocuments.org/doc_vishhnu/vishnupanchAyudh.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/ranganatha_stotram.py

@@ -0,0 +1,92 @@
+from modules.config.categories import ScriptureCategoryConstants
+from modules.google_drive.google_drive_utils import convert_drive_url_to_direct
+
+ranganatha_stotram_config = {
+    "name": "ranganatha_stotram",
+    "title": "Ranganatha Stotram",
+    "banner_url": convert_drive_url_to_direct("https://drive.google.com/file/d/1uLPo1Upx1vLJ6InxqOEMspAOz4UT1F4r/view?usp=drive_link"),
+    "category": ScriptureCategoryConstants.SRI_SOOKTHI,
+    "output_dir": "./outputs/ranganatha_stotram",
+    "collection_name": "ranganatha_stotram",
+    "collection_embedding_fn": "openai",
+
+    "unit": "slokam",
+    "unit_field": "_global_index",
+
+    "field_mapping": {
+        "text": "lyrics_sa",
+        "chapter_name": "chapter_name",
+        "unit_index": "_global_index",
+        "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('slokam_index')}",
+        "author": lambda doc: "Sri Parasara Bhattar",
+    },
+
+    "metadata_fields": [
+        {
+            "name": "chapter_name",
+            "datatype": "str",
+            "label": "Chapter Name",
+            "description": "Chapter title",
+            "show_as_filter": True,
+            "component": "dropdown",
+            "lov": lambda: ["SLOKAM", "THANIYAN"],
+        },
+        {
+            "name": "_global_index",
+            "datatype": "int",
+            "label": "Global Index",
+            "description": "Absolute verse index",
+            "show_as_filter": True,
+            "is_unique": True,
+        },
+        {
+            "name": "chapter_index",
+            "datatype": "int",
+            "label": "Chapter Index",
+            "description": "Chapter index",
+        },
+        {
+            "name": "slokam_index",
+            "datatype": "int",
+            "label": "Verse Number (in chapter)",
+            "description": "Sequential verse index inside the chapter",
+            "show_as_filter": True,
+        },
+        {
+            "name": "lyrics_sa",
+            "datatype": "str",
+            "label": "Sanskrit Verse",
+            "description": "Original verse text in Sanskrit",
+            "show_as_filter": True,
+        },
+    ],
+
+    "pdf_path": "./data/ranganatha_stotram.pdf",
+
+    "source": "sanskritdocuments.org",
+    "language": "san",
+    "example_labels": [],
+    "examples": [],
+    "llm_hints": [],
+
+    "credits": {
+        "art": [
+            {
+                "name": "Sri TNCN",
+                "url": "https://www.linkedin.com/in/narashiman-chakravarthy-54326111/",
+                "role": "Illustrator",
+                "notes": "Original artwork used with permission",
+                "photo_url": "https://drive.google.com/uc?export=download&id=1D4Vo-Mvkp1RdZR6whUkcjvNBIW1yHSUE",
+            }
+        ],
+        "data": [
+        {
+                "name": "sanskritdocuments.org",
+                "url": "https://sanskritdocuments.org/doc_vishhnu/ranganAthastotram.html",
+                "role": "Data Provider"
+        }
+],
+        "audio": [],
+        "video": [],
+    },
+}

modules/config/thiruvaimozhi_nootrandhadhi.py

@@ -23,6 +23,7 @@ thiruvaimozhi_nootrandhadhi_config = {
         "unit_index": "_global_index",
         "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('relative_verse_number')}",
         "word_by_word_native": "word_by_word_en",
+        "author": lambda doc: f"Manavala Mamunigal",
         "translation": lambda doc: "\n\n".join(
             filter(None, [
                 f"TAMIL:\n{doc.get('translation_ta')}" if doc.get('translation_ta') else None,

modules/config/upadesa_rathnamalai.py

@@ -21,6 +21,7 @@ upadesa_rathnamalai_config = {
         "text": "pasuram",
         "chapter_name": "chapter_name",
         "unit_index": "_global_index",
+        "author": lambda doc: f"Manavala Mamunigal",
         "relative_path": lambda doc: f"{doc.get('chapter_name')} — {doc.get('relative_verse_number')}",
         "translation": "translation",
     },