Fix delete all persistence - use bulk delete for ChromaDB metadata and documents

app.py

@@ -856,50 +856,46 @@ def delete_all_endpoint(mode):
     deleted_count = 0
     
     try:
-        # Step 1: Get all items to delete from JSON
+        # Step 1: Count items before deletion (from both sources)
         user_data = _load_user_data(user_api_key, mode)
-        
-        # Step 2: Also get items from ChromaDB (in case they're not in JSON)
         chroma_data = rag_core.load_all_metadata_from_chroma(user_api_key, mode)
         
-        # Combine all item IDs
-        all_item_ids = set()
-        for item in user_data:
-            if item.get('id'):
-                all_item_ids.add(item.get('id'))
-        if chroma_data:
-            for item in chroma_data:
-                if item.get('id'):
-                    all_item_ids.add(item.get('id'))
+        # Get count from whichever source has more
+        deleted_count = max(len(user_data), len(chroma_data) if chroma_data else 0)
+        
+        if deleted_count == 0:
+            return jsonify({
+                'success': True,
+                'deleted_count': 0,
+                'message': f'No {mode} to delete'
+            })
         
-        deleted_count = len(all_item_ids)
+        print(f"Starting deletion of {deleted_count} {mode}...")
         
-        # Step 3: Clear JSON file
+        # Step 2: Clear JSON file
         _save_user_data(user_api_key, mode, [])
+        print(f"Cleared JSON file for {mode}")
         
-        # Step 4: Delete from ChromaDB (RAG knowledge base)
-        for item_id in all_item_ids:
-            try:
-                if mode == 'cards':
-                    rag_core.remove_document_from_knowledge_base(user_api_key, item_id, mode)
-                    rag_core.delete_metadata_from_chroma(user_api_key, mode, item_id)
-                elif mode == 'brochures':
-                    rag_core.remove_document_from_knowledge_base(user_api_key, f"{item_id}_contacts", mode)
-                    rag_core.remove_document_from_knowledge_base(user_api_key, f"{item_id}_info", mode)
-                    rag_core.delete_metadata_from_chroma(user_api_key, mode, item_id)
-            except Exception as e:
-                print(f"ChromaDB deletion warning for {item_id}: {e}")
+        # Step 3: Delete ALL metadata from ChromaDB (bulk delete)
+        metadata_deleted = rag_core.delete_all_metadata_from_chroma(user_api_key, mode)
+        print(f"Deleted {metadata_deleted} metadata records from ChromaDB")
+        
+        # Step 4: Delete ALL document chunks from ChromaDB (bulk delete)
+        docs_deleted = rag_core.delete_all_documents_from_chroma(user_api_key, mode)
+        print(f"Deleted {docs_deleted} document chunks from ChromaDB")
         
         # Step 5: Delete from SQL Database
         user_hash = hashlib.sha256(user_api_key.encode()).hexdigest()
         if mode == 'cards':
-            BusinessCard.query.filter_by(user_hash=user_hash).delete()
+            db_deleted = BusinessCard.query.filter_by(user_hash=user_hash).delete()
+            print(f"Deleted {db_deleted} business cards from SQL database")
         elif mode == 'brochures':
             # Delete all brochures and their contacts (cascade)
-            Brochure.query.filter_by(user_hash=user_hash).delete()
+            db_deleted = Brochure.query.filter_by(user_hash=user_hash).delete()
+            print(f"Deleted {db_deleted} brochures from SQL database")
         
         db.session.commit()
-        print(f"Successfully deleted all {deleted_count} {mode} from database")
+        print(f"Successfully deleted all {deleted_count} {mode} from all storage layers")
         
         return jsonify({
             'success': True,

rag_core.py

@@ -813,6 +813,85 @@ def delete_metadata_from_chroma(user_api_key, mode, document_id):
         return False
 
 
+def delete_all_metadata_from_chroma(user_api_key, mode):
+    """
+    Delete ALL metadata documents from ChromaDB for a user/mode.
+    This is used for the 'delete all' feature.
+    Returns the count of deleted items.
+    """
+    if not _rag_system_available:
+        print("RAG: System not available, cannot delete metadata from ChromaDB")
+        return 0
+    
+    try:
+        collection = _get_or_create_collection(user_api_key, mode)
+        
+        # Get all metadata document IDs
+        results = collection.get(
+            where={"type": "metadata"},
+            include=["metadatas"]
+        )
+        
+        if not results or not results['ids']:
+            print(f"RAG: No metadata to delete for {mode}")
+            return 0
+        
+        deleted_count = len(results['ids'])
+        
+        # Delete all metadata documents
+        collection.delete(ids=results['ids'])
+        print(f"RAG: Deleted {deleted_count} metadata records from ChromaDB for {mode}")
+        
+        return deleted_count
+    except Exception as e:
+        print(f"RAG: Error deleting all metadata from ChromaDB: {e}")
+        import traceback
+        traceback.print_exc()
+        return 0
+
+
+def delete_all_documents_from_chroma(user_api_key, mode):
+    """
+    Delete ALL document chunks from ChromaDB for a user/mode.
+    This removes RAG knowledge base entries.
+    Returns the count of deleted chunks.
+    """
+    if not _rag_system_available:
+        print("RAG: System not available, cannot delete documents from ChromaDB")
+        return 0
+    
+    try:
+        collection = _get_or_create_collection(user_api_key, mode)
+        
+        # Get all document chunk IDs
+        results = collection.get(
+            where={"type": "document_chunk"},
+            include=["metadatas"]
+        )
+        
+        if not results or not results['ids']:
+            print(f"RAG: No document chunks to delete for {mode}")
+            return 0
+        
+        deleted_count = len(results['ids'])
+        
+        # Delete all document chunks
+        collection.delete(ids=results['ids'])
+        print(f"RAG: Deleted {deleted_count} document chunks from ChromaDB for {mode}")
+        
+        # Clear keyword index
+        if mode in keyword_indexes and user_api_key in keyword_indexes[mode]:
+            keyword_indexes[mode][user_api_key] = {"documents": {}, "vocabulary": {}, "entities": {}}
+            _save_keyword_index(user_api_key, mode)
+        
+        return deleted_count
+    except Exception as e:
+        print(f"RAG: Error deleting all documents from ChromaDB: {e}")
+        import traceback
+        traceback.print_exc()
+        return 0
+
+
 def update_metadata_in_chroma(user_api_key, mode, document_id, field, value, contact_id=None):
     """
     Update a specific field in the metadata stored in ChromaDB.