more embeddings path tweaks

scripts/browse_hf_data.py

@@ -0,0 +1,31 @@
+import os
+import streamlit as st
+from pathlib import Path
+
+def list_files_in_directory(directory):
+    """List all files in the given directory and its subdirectories."""
+    files = []
+    for root, dirs, filenames in os.walk(directory):
+        for filename in filenames:
+            files.append(os.path.join(root, filename))
+    return files
+
+def main():
+    st.title("Embeddings File Browser")
+
+    # Directory to browse
+    directory = "/data"  # Persistent storage directory
+    st.write(f"Browsing directory: {directory}")
+
+    # List files
+    files = list_files_in_directory(directory)
+
+    if files:
+        st.write("Files found:")
+        for file in files:
+            st.write(file)
+    else:
+        st.write("No files found in the directory.")
+
+if __name__ == "__main__":
+    main()

src/generator/generator.py

@@ -82,7 +82,7 @@ class LyricGenerator:
             if Settings.is_huggingface():
                 print("HuggingFace environment detected, setting up embeddings...")
                 self._setup_embeddings_from_hf()
-                chroma_dir = Path("/data/chroma")  # Assuming /data is the root for persistent storage
+                chroma_dir = Path("/data/processed/embeddings/chroma")
             else:
                 print("Local environment detected")
                 print(f"Base directory: {Settings.BASE_DIR}")
@@ -130,8 +130,7 @@ class LyricGenerator:
                 for item in chroma_dir.glob('**/*'):
                     print(f"  {item}")
                     if item.is_file():
-                        print(
-                            f"    Size: {item.stat().st_size / (1024*1024):.2f} MB")
+                        print(f"    Size: {item.stat().st_size / (1024*1024):.2f} MB")
 
                 raise RuntimeError(
                     "Chroma DB is empty. Please ensure embeddings "