Upload folder using huggingface_hub

src/generator/generator.py

@@ -106,23 +106,28 @@ class LyricGenerator:
         """Download and setup embeddings from HuggingFace dataset"""
         print("\n=== Setting up embeddings from HuggingFace dataset ===")
         try:
-            # Create data directory if it doesn't exist
-            data_dir = Path("/data")
-            data_dir.mkdir(exist_ok=True)
-            
-            # First find the snapshot directory
-            snapshot_pattern = "**/datasets--*--*/snapshots/*/chroma"
-            snapshots = list(data_dir.glob(snapshot_pattern))
-            if not snapshots:
-                raise RuntimeError("No snapshot directories found")
-            
-            # Use most recent snapshot
-            chosen_path = max(snapshots, key=lambda p: p.stat().st_mtime)
-            print(f"Using snapshot directory: {chosen_path}")
-            
+            # Force fresh download of the dataset to ensure latest HNSW index
+            print("Downloading latest dataset snapshot...")
+            snapshot_path = snapshot_download(
+                repo_id=Settings.HF_DATASET,
+                repo_type="dataset",
+                token=Settings.HF_TOKEN,
+                cache_dir="/data",
+            )
+            chroma_path = Path(snapshot_path) / "chroma"
+            print(f"Downloaded snapshot to: {snapshot_path}")
+
+            if not chroma_path.exists():
+                raise RuntimeError(f"chroma/ not found in snapshot at {chroma_path}")
+
             # Set the chroma directory
-            self.chroma_dir = chosen_path
+            self.chroma_dir = chroma_path
             print(f"Chroma directory set to: {self.chroma_dir}")
+
+            # Log index files for debugging
+            for f in sorted(chroma_path.rglob("*")):
+                if f.is_file():
+                    print(f"  {f.name}: {f.stat().st_size / (1024*1024):.1f} MB")
             
         except Exception as e:
             print(f"\n=== Error in _setup_embeddings_from_hf ===")