Update main.py

main.py

@@ -514,60 +514,66 @@ async def chat(request: ChatRequest):
         raise HTTPException(status_code=500, detail=str(e))
 
 def process_heavy_files_background(space_id: str, saved_file_paths: List[Dict]):
-    """Runs in the background so the HTTP request doesn't timeout"""
+    """Runs in the background, processing and saving ONE file at a time."""
     try:
         initialize_space(space_id)
         processor = DocumentProcessor()
-        all_chunks = []
-        processed_files = []
 
         for file_info in saved_file_paths:
-            file_path = Path(file_info['path'])
-            filename = file_info['name']
-            
-            # The heavy CPU work
-            file_data = processor.process_file(file_path)
-            chunks = processor.chunk_text(file_data['content'], chunk_size=512, overlap=50, semantic=True)
-            
-            for idx, chunk in enumerate(chunks):
-                all_chunks.append({
-                    'content': chunk,
-                    'metadata': {
+            try:
+                file_path = Path(file_info['path'])
+                filename = file_info['name']
+                
+                print(f"Processing: {filename}...")
+                
+                # 1. Process just this one file
+                file_data = processor.process_file(file_path)
+                chunks = processor.chunk_text(file_data['content'], chunk_size=512, overlap=50, semantic=True)
+                
+                file_chunks = []
+                for idx, chunk in enumerate(chunks):
+                    file_chunks.append({
+                        'content': chunk,
+                        'metadata': {
+                            'filename': filename,
+                            'chunk_index': idx,
+                            'total_chunks': len(chunks),
+                            'source_type': file_data['format']
+                        }
+                    })
+                
+                # 2. Upload to Qdrant immediately (This clears the RAM for the next file!)
+                if file_chunks:
+                    texts = [chunk['content'] for chunk in file_chunks]
+                    metadatas = [chunk['metadata'] for chunk in file_chunks]
+                    # Make UUID unique to the file to prevent collisions
+                    ids = [f"{space_id}_{filename}_{idx}_{uuid.uuid4().hex[:8]}" for idx in range(len(file_chunks))]
+                    
+                    batch_size = 100
+                    for i in range(0, len(texts), batch_size):
+                        vector_db.add_documents(
+                            texts[i:i + batch_size], 
+                            metadatas[i:i + batch_size], 
+                            ids[i:i + batch_size]
+                        )
+
+                # 3. Save metadata directly to MongoDB so it appears in Flutter instantly
+                if MONGO_URI:
+                    files_collection.insert_one({
                         'filename': filename,
-                        'chunk_index': idx,
-                        'total_chunks': len(chunks),
-                        'source_type': file_data['format']
-                    }
-                })
-            
-            processed_files.append({
-                'filename': filename,
-                'space_id': space_id, # Link it to the space
-                'chunks': len(chunks),
-                'processed_at': datetime.now().isoformat()
-            })
-
-        # Upload to Qdrant
-        if all_chunks:
-            texts = [chunk['content'] for chunk in all_chunks]
-            metadatas = [chunk['metadata'] for chunk in all_chunks]
-            ids = [f"{space_id}_{idx}_{uuid.uuid4().hex[:8]}" for idx in range(len(all_chunks))]
-            
-            # Batch size set to 100 for Qdrant Cloud limits
-            batch_size = 100
-            for i in range(0, len(texts), batch_size):
-                vector_db.add_documents(
-                    texts[i:i + batch_size], 
-                    metadatas[i:i + batch_size], 
-                    ids[i:i + batch_size]
-                )
-
-        # Save metadata directly to MongoDB
-        if processed_files and MONGO_URI:
-            files_collection.insert_many(processed_files)
+                        'space_id': space_id,
+                        'chunks': len(chunks),
+                        'processed_at': datetime.now().isoformat()
+                    })
+                    
+                print(f"Successfully finished: {filename}")
+                    
+            except Exception as file_e:
+                # If ONE file has a corrupted page, skip it but KEEP GOING for the rest!
+                print(f"Failed to process file {file_info['name']}: {file_e}")
 
     except Exception as e:
-        print(f"Background processing failed: {e}")
+        print(f"Background worker completely crashed: {e}")
 
 @app.post("/api/spaces/{space_id}/upload")
 async def upload_files(