Trying to fix this HF dataset fiasco

src/generator/generator.py

@@ -47,12 +47,12 @@ class LyricGenerator:
         """Download and setup embeddings from HuggingFace dataset"""
         print("\n=== Setting up embeddings from HuggingFace dataset ===")
         try:
-            # Load dataset directly into the desired directory
+            # Load the latest version of the dataset into the desired directory
             dataset = load_dataset("SongLift/LyrGen2_DB", split='train', cache_dir="/data")
             print("Dataset loaded successfully into cache directory.")
 
             # Verify the contents of the cache directory
-            chroma_dir = Path("/data")
+            chroma_dir = Path("/data/chroma")
             print(f"Persistent storage contents: {list(chroma_dir.glob('**/*'))}")
 
         except Exception as e:
@@ -66,17 +66,16 @@ class LyricGenerator:
         try:
             print("\n=== Loading Embeddings ===")
 
-            # If in HuggingFace environment, ensure embeddings are set up
+            # Determine the environment and set paths accordingly
             if Settings.is_huggingface():
                 print("HuggingFace environment detected, setting up embeddings...")
                 self._setup_embeddings_from_hf()
+                chroma_dir = Path("/data/chroma")  # Assuming /data is the root for persistent storage
             else:
                 print("Local environment detected")
                 print(f"Base directory: {Settings.BASE_DIR}")
+                chroma_dir = Path("/data/processed/embeddings/chroma")  # Local environment path
 
-            print(f"\nLoading vector store from: {self.embeddings_dir}")
-            # Check Chroma directory structure
-            chroma_dir = Path("/data/chroma")
             print(f"Checking Chroma directory: {chroma_dir}")
             print(f"Absolute path: {chroma_dir.absolute()}")