Upload folder using huggingface_hub

data/dropbox/scriptures/sri_stavam/video_metadata.json

@@ -0,0 +1,62 @@
+[
+    {
+        "scripture": "sri_stavam",
+        "global_index": 2,
+        "video_url": "https://www.youtube.com/watch?v=u2MUPrE7Zx8",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "sri_stavam",
+        "global_index": 3,
+        "video_url": "https://www.youtube.com/watch?v=s-aIwjzmEbs",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "sri_stavam",
+        "global_index": 4,
+        "video_url": "https://www.youtube.com/watch?v=WjOfN-GiZXE",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "sri_stavam",
+        "global_index": 5,
+        "video_url": "https://www.youtube.com/watch?v=ivPWtir9JGY",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "sri_stavam",
+        "global_index": 6,
+        "video_url": "https://www.youtube.com/watch?v=_QBqGwpjOO0",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "sri_stavam",
+        "global_index": 7,
+        "video_url": "https://www.youtube.com/watch?v=5K36X0FhY8Y",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "sri_stavam",
+        "global_index": 8,
+        "video_url": "https://www.youtube.com/watch?v=FLhSyUkMPGw",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "sri_stavam",
+        "global_index": 9,
+        "video_url": "https://www.youtube.com/watch?v=q-Vf7T1CPME",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "sri_stavam",
+        "global_index": 10,
+        "video_url": "https://www.youtube.com/watch?v=TEpI4koRvRI",
+        "type": "upanyasam"
+    },
+    {
+        "scripture": "sri_stavam",
+        "global_index": 11,
+        "video_url": "https://www.youtube.com/watch?v=9ievpAujqnw",
+        "type": "upanyasam"
+    }
+]

db.py

@@ -38,6 +38,10 @@ class SanatanDatabase:
             metadatas=metadatas,
         )
 
+    def get(self, collection_name: str, where, n_results=5):
+        collection = self.chroma_client.get_or_create_collection(name=collection_name)
+        return collection.get(where=where, limit=n_results)
+
     def fetch_random_data(
         self,
         collection_name: str,
@@ -77,7 +81,7 @@ class SanatanDatabase:
             metadatas=[metas[i] for i in indices],
         )
 
-    def fetch_first_match( 
+    def fetch_first_match(
         self, collection_name: str, metadata_where_clause: MetadataWhereClause = None
     ):
         """This version is created to support the browse module with fallback regex matching"""
@@ -87,6 +91,7 @@ class SanatanDatabase:
             s = unicodedata.normalize("NFD", s)
             s = "".join(ch for ch in s if not unicodedata.combining(ch))
             return s
+
         logger.info(
             "getting first matching verses from [%s] | metadata_where_clause = %s",
             collection_name,
@@ -122,7 +127,8 @@ class SanatanDatabase:
 
         # find filters with $eq string type
         regex_filters = [
-            f for f in metadata_where_clause.filters
+            f
+            for f in metadata_where_clause.filters
             if f.metadata_search_operator == "$eq" and isinstance(f.metadata_value, str)
         ]
 
@@ -137,7 +143,7 @@ class SanatanDatabase:
             ok = True
             for f in regex_filters:
                 field_val = str(meta.get(f.metadata_field, ""))
-                
+
                 # Normalize both the stored field and the search value
                 norm_val = normalize_for_match(field_val)
                 norm_query = normalize_for_match(f.metadata_value)
@@ -149,7 +155,6 @@ class SanatanDatabase:
             if ok:
                 matched_indices.append(i)
 
-
         if not matched_indices:
             logger.warning("Regex fallback also found no matches.")
             return chromadb.GetResult(ids=[], documents=[], metadatas=[])
@@ -715,4 +720,4 @@ class SanatanDatabase:
     def delete_taniyans_in_divya_prabandham(self):
         nalayiram_helper.delete_taniyan(
             self.chroma_client.get_collection("divya_prabandham")
-        )
+        )

tests/test_yt_search.py

@@ -0,0 +1,21 @@
+import json
+import logging
+
+from db import SanatanDatabase
+from metadata import MetadataFilter, MetadataWhereClause
+
+if __name__ == "__main__":
+    logging.basicConfig()
+    collection_name = "yt_metadata"
+    database = SanatanDatabase()
+    print("count = ", database.count(collection_name))
+    response = database.fetch_first_match(
+        collection_name=collection_name,
+        metadata_where_clause=MetadataWhereClause(filters=[
+            # MetadataFilter(metadata_field="channel_id", metadata_search_operator="$eq", metadata_value="UCqa48rNanVRKmG4qxl-YmEQ"),
+            MetadataFilter(metadata_field="channel_title", metadata_search_operator="$eq", metadata_value="sookthi"),
+            MetadataFilter(metadata_field="video_title", metadata_search_operator="$eq", metadata_value="krishna mangalam")
+
+        ])
+    )
+    print(json.dumps(response, indent=1, ensure_ascii=False))