Checkpoint! local working probably. new embeddings, added to readme, additional scripts, updated process_lyrics and upload_embeddings and added some testscripts.

config/settings.py

@@ -29,7 +29,7 @@ class Settings:
     LLM_MODEL = "gpt-4"
     
     # ChromaDB Settings
-    CHROMA_COLLECTION_NAME = "langchain"
+    CHROMA_COLLECTION_NAME = "lyrics_v1"
     
     @classmethod
     def is_huggingface(cls) -> bool:

docs/PROJECT_README.md

@@ -70,4 +70,56 @@ python scripts/test_embeddings.py
 python scripts/test_semantic.py
 ```
 !
-For troubleshooting and known issues, see TROUBLESHOOTING.md 
+For troubleshooting and known issues, see TROUBLESHOOTING.md 
+
+## Data Processing & Loading Workflow
+
+### 1. Lyrics Processing (`scripts/process_lyrics.py`)
+- Handles initial lyrics processing and embedding creation
+- Features:
+  - Batch processing with rate limiting
+  - Recursive text splitting (300 chars, 75 overlap)
+  - Automatic retry on API limits
+  - Comprehensive metadata tracking
+  - Progress monitoring with tqdm
+
+### 2. Data Loading (`src/utils/data_loader.py`)
+- Manages raw lyrics loading and cleaning
+- Features:
+  - Multi-encoding support (utf-8, latin-1, cp1252)
+  - Section marker detection ([Verse], [Chorus], etc.)
+  - Intelligent line breaking
+  - Metadata preservation (artist, song title)
+  - Robust validation checks
+
+### 3. Workflow Steps
+```bash
+# 1. Load and clean lyrics
+python scripts/process_lyrics.py
+# Creates cleaned, chunked documents with metadata
+
+# 2. Generate embeddings
+# (Handled automatically by process_lyrics.py)
+# Stores in data/processed/embeddings/chroma
+
+# 3. Upload to HuggingFace
+python scripts/upload_embeddings.py
+# Verifies and uploads to SongLift/LyrGen2_DB
+```
+
+### Data Flow
+```
+Raw Lyrics (txt)
+    ↓
+Data Loader (cleaning)
+    ↓
+Text Splitter (chunking)
+    ↓
+OpenAI Embeddings
+    ↓
+ChromaDB Storage
+    ↓
+HuggingFace Dataset
+```
+
+Each step includes validation and error handling to ensure data integrity. The process is designed to be resumable and maintains consistency between local and deployed environments.

scripts/check_collections.py

@@ -0,0 +1,63 @@
+import sqlite3
+from pathlib import Path
+import sys
+
+# Add project root to path
+project_root = Path(__file__).parent.parent
+sys.path.append(str(project_root))
+
+from config.settings import Settings
+
+def check_collections():
+    """Check collections in ChromaDB"""
+    chroma_path = Settings.get_chroma_path()
+    sqlite_file = chroma_path / "chroma.sqlite3"
+    
+    print(f"\nExamining: {sqlite_file}")
+    conn = sqlite3.connect(sqlite_file)
+    cursor = conn.cursor()
+    
+    try:
+        # List collections
+        cursor.execute("SELECT name, id FROM collections;")
+        collections = cursor.fetchall()
+        
+        print("\nCollections found:")
+        for name, coll_id in collections:
+            print(f"Name: {name}")
+            print(f"ID: {coll_id}")
+            
+            # Get count of embeddings using correct schema
+            try:
+                cursor.execute("""
+                    SELECT COUNT(*) 
+                    FROM embeddings e
+                    JOIN segments s ON e.segment_id = s.id
+                    WHERE s.collection = ?
+                """, (coll_id,))
+                count = cursor.fetchone()[0]
+                print(f"Embeddings count: {count}")
+            except sqlite3.OperationalError as e:
+                print(f"Could not get count: {e}")
+            
+            # Get metadata using correct schema
+            try:
+                cursor.execute("""
+                    SELECT key, str_value 
+                    FROM collection_metadata 
+                    WHERE collection_id = ?
+                """, (coll_id,))
+                metadata = cursor.fetchall()
+                if metadata:
+                    print("Collection Metadata:")
+                    for key, value in metadata:
+                        print(f"  {key}: {value}")
+            except sqlite3.OperationalError as e:
+                print(f"Could not get metadata: {e}")
+            print()
+    
+    finally:
+        conn.close()
+
+if __name__ == "__main__":
+    check_collections() 

scripts/cleanup_old_embeddings.py

@@ -0,0 +1,53 @@
+import sqlite3
+import shutil
+from pathlib import Path
+import sys
+
+# Add project root to path
+project_root = Path(__file__).parent.parent
+sys.path.append(str(project_root))
+
+from config.settings import Settings
+
+
+def cleanup_old_embeddings():
+    """Clean up old embedding directories"""
+    chroma_path = Settings.get_chroma_path()
+    
+    # List all UUID directories
+    collection_dirs = list(chroma_path.glob("*-*-*-*-*"))
+    
+    print("\nFound collection directories:")
+    for dir_path in collection_dirs:
+        print(f"- {dir_path.name}")
+    
+    # Get current collection info from database
+    sqlite_file = chroma_path / "chroma.sqlite3"
+    conn = sqlite3.connect(sqlite_file)
+    cursor = conn.cursor()
+    
+    try:
+        # Get all active collection IDs
+        cursor.execute("SELECT id FROM collections")
+        active_ids = {row[0] for row in cursor.fetchall()}
+        print("\nActive collection IDs:")
+        for id in active_ids:
+            print(f"- {id}")
+        
+        # Find directories that don't match any active collection
+        for dir_path in collection_dirs:
+            if dir_path.name not in active_ids:
+                print(f"\nFound unused collection directory: {dir_path.name}")
+                response = input("Delete this directory? (y/N): ")
+                if response.lower() == 'y':
+                    shutil.rmtree(dir_path)
+                    print(f"Deleted: {dir_path}")
+                else:
+                    print("Skipped deletion")
+    
+    finally:
+        conn.close()
+
+
+if __name__ == "__main__":
+    cleanup_old_embeddings() 

scripts/process_lyrics.py

@@ -3,6 +3,8 @@ import sys
 import time
 from datetime import datetime
 from pathlib import Path
+import sqlite3
+import shutil
 
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import Chroma
@@ -35,7 +37,9 @@ class LyricsProcessor:
         self.output_dir = Path(output_dir)
         self.batch_size = batch_size
         self.embeddings = OpenAIEmbeddings()
-        self.collection_name = "langchain"
+        self.collection_name = Settings.CHROMA_COLLECTION_NAME
+        
+        print(f"Using collection name: {self.collection_name}")
         
         # Configure text splitter for lyrics
         self.text_splitter = RecursiveCharacterTextSplitter(
@@ -83,6 +87,32 @@ class LyricsProcessor:
         print("Validating configuration...")
         self.validate_text_splitter()
         
+        # Check for existing collection
+        chroma_dir = Path(self.output_dir) / "chroma"
+        if chroma_dir.exists():
+            sqlite_file = chroma_dir / "chroma.sqlite3"
+            if sqlite_file.exists():
+                try:
+                    conn = sqlite3.connect(sqlite_file)
+                    cursor = conn.cursor()
+                    cursor.execute("SELECT name FROM collections WHERE name = ?", 
+                                 (self.collection_name,))
+                    if cursor.fetchone():
+                        response = input(
+                            f"\nWarning: Collection '{self.collection_name}' already exists.\n"
+                            "Do you want to delete and recreate? (y/N): "
+                        )
+                        if response.lower() != 'y':
+                            print("Aborting.")
+                            return
+                        print("Removing existing collection...")
+                        shutil.rmtree(chroma_dir)
+                        chroma_dir.mkdir(parents=True)
+                    conn.close()
+                except Exception as e:
+                    print(f"Error checking existing collection: {e}")
+                    print("Continuing with processing...")
+        
         # Load all lyrics documents
         print("Loading lyrics files...")
         documents = self.loader.load_lyrics()

scripts/upload_embeddings.py

@@ -2,8 +2,7 @@
 import sys
 from pathlib import Path
 from huggingface_hub import HfApi
-import os
-import shutil
+import sqlite3
 
 # Add parent directory to path to locate 'config' module
 sys.path.append(str(Path(__file__).parent.parent))
@@ -31,41 +30,86 @@ def verify_space_access():
         print(f"Error verifying space access: {str(e)}")
         return False
 
-def main():
-    """Upload embeddings directory to HuggingFace Space using HfApi.upload_folder"""
-    if not Settings.HF_TOKEN:
-        raise ValueError("HF_TOKEN not found in environment variables")
-        
-    print("Verifying Space access...")
-    if not verify_space_access():
-        raise RuntimeError("Failed to verify Space access")
-    
-    print("Starting upload process...")
+def verify_and_upload():
+    """Verify local embeddings and upload to HuggingFace"""
+    local_chroma = Settings.get_chroma_path()
     
-    # Calculate total size of embeddings
-    total_size = sum(
-        f.stat().st_size for f in Settings.EMBEDDINGS_DIR.glob('**/*') if f.is_file()
-    )
-    print(f"Found embeddings: {total_size / 1024 / 1024:.2f} MB")
+    # Verify local files first
+    print("\nVerifying local embeddings...")
+    sqlite_file = local_chroma / "chroma.sqlite3"
+    if not sqlite_file.exists():
+        raise RuntimeError(f"SQLite database not found at {sqlite_file}")
     
-    api = HfApi(token=Settings.HF_TOKEN)
+    # Get collection info
+    conn = sqlite3.connect(sqlite_file)
+    cursor = conn.cursor()
     
-    print(f"Uploading to Space: {Settings.HF_SPACE}...")
     try:
-        # Upload using upload_folder
-        api.upload_folder(
-            folder_path=str(Settings.EMBEDDINGS_DIR),
-            repo_id=Settings.HF_SPACE,
-            repo_type="space",
-            ignore_patterns=["*.pyc", "__pycache__/", ".DS_Store"]
+        # Verify collection
+        cursor.execute("SELECT name, id FROM collections;")
+        collections = cursor.fetchall()
+        
+        if not collections:
+            raise RuntimeError("No collections found in database")
+            
+        print("\nCollections in database:")
+        for name, coll_id in collections:
+            # Get embeddings count
+            cursor.execute("""
+                SELECT COUNT(*) 
+                FROM embeddings e
+                JOIN segments s ON e.segment_id = s.id
+                WHERE s.collection = ?
+            """, (coll_id,))
+            count = cursor.fetchone()[0]
+            print(f"Collection: {name}")
+            print(f"ID: {coll_id}")
+            print(f"Embeddings: {count}")
+        
+        # Verify expected collection exists
+        collection_names = [c[0] for c in collections]
+        if Settings.CHROMA_COLLECTION_NAME not in collection_names:
+            raise RuntimeError(
+                f"Expected collection '{Settings.CHROMA_COLLECTION_NAME}' not found. "
+                f"Found: {collection_names}"
+            )
+        
+        # Calculate total size
+        total_size = sum(
+            f.stat().st_size for f in local_chroma.glob('**/*') if f.is_file()
         )
-        print("Upload complete!")
-    except Exception as e:
-        print(f"Error during upload: {str(e)}")
-        if "storage" in str(e).lower():
-            print("\nNOTE: This might be a Git LFS limitation.")
-            print("Consider splitting the upload into smaller chunks or using the Hugging Face web interface.")
-        raise
+        print(f"\nTotal size to upload: {total_size / (1024*1024):.2f} MB")
+        
+        # Confirm before upload
+        response = input("\nReady to upload to HuggingFace. Continue? (y/N): ")
+        if response.lower() != 'y':
+            print("Upload cancelled.")
+            return
+        
+        print("\nUploading to HuggingFace...")
+        api = HfApi(token=Settings.HF_TOKEN)
+        
+        try:
+            # Upload files to the existing repo
+            api.upload_folder(
+                folder_path=str(local_chroma),
+                repo_id=Settings.HF_DATASET,
+                repo_type="dataset",
+                path_in_repo="chroma",
+                ignore_patterns=["*.pyc", "__pycache__/", ".DS_Store"]
+            )
+            print("Upload complete!")
+            
+        except Exception as e:
+            print(f"Error during upload: {e}")
+            print("\nNOTE: If this was a storage limitation error:")
+            print("1. Verify Git LFS is enabled")
+            print("2. Check HuggingFace storage quota")
+            print("3. Ensure all files are under size limits")
+            raise
+            
+    finally:
+        conn.close()
 
 if __name__ == "__main__":
-    main()
+    verify_and_upload()