Update main.py

main.py

@@ -12,7 +12,7 @@ Secure multi-user API with:
 import os
 import asyncio
 from typing import List, Optional, Dict
-from fastapi import FastAPI, UploadFile, File, HTTPException, Depends
+from fastapi import FastAPI, UploadFile, File, HTTPException, Depends, BackgroundTasks
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
 from threading import Lock
@@ -20,6 +20,7 @@ from rag_engine import RAGEngine
 from hf_storage import create_hf_storage_manager
 from auth import get_current_user
 from user_management import create_hf_user_manager
+from job_worker import job_manager
 from dotenv import load_dotenv
 
 # ============================================
@@ -167,6 +168,13 @@ async def startup_event():
     
     print("✅ Server ready (per-user lazy loading)")
 
+    # Start persistent job manager
+    try:
+        job_manager.start()
+        print("🔁 Background job manager started")
+    except Exception as e:
+        print(f"⚠️ Failed to start job manager: {e}")
+
 
 # ============================================
 # ENDPOINTS
@@ -181,7 +189,8 @@ async def health_check():
 @app.post("/upload", response_model=UploadResponse)
 async def upload_pdf(
     file: UploadFile = File(...),
-    user_id: str = Depends(get_current_user)
+    user_id: str = Depends(get_current_user),
+    background_tasks: BackgroundTasks = None
 ):
     """
     Upload PDF for authenticated user.
@@ -194,59 +203,45 @@ async def upload_pdf(
     if file.content_type and file.content_type not in ['application/pdf']:
         raise HTTPException(400, "Invalid MIME type")
     
-    # Read content
+    # Read content quickly and persist to a temp file, then enqueue a job for background processing
     content = await file.read()
-    
+
     # Size limit (10MB)
     if len(content) > 10 * 1024 * 1024:
         raise HTTPException(413, "File too large (max 10MB)")
-    
+
     try:
-        engine = await rag_manager.get_engine(user_id)
-        user_lock = rag_manager.get_user_lock(user_id)
-        
-        with user_lock:
-            result = await asyncio.to_thread(
-                engine.upload_document,
-                filename=file.filename,
-                file_content=content,
-                action="auto"
-            )
-            
-            if result["status"] == "success":
-                # Save PDF to user's uploaded_pdfs folder
-                base_dir = os.path.dirname(os.path.abspath(__file__))
-                user_pdfs_dir = os.path.join(base_dir, "users", user_id, "uploaded_pdfs")
-                os.makedirs(user_pdfs_dir, exist_ok=True)
-                
-                pdf_path = os.path.join(user_pdfs_dir, file.filename)
-                with open(pdf_path, "wb") as f:
-                    f.write(content)
-                
-                await asyncio.to_thread(
-                    hf_storage.push_storage_to_hf,
-                    user_id,
-                    f"Upload {file.filename}"
-                )
-                
-                print(f"✅ Upload success for user {user_id}: {file.filename}")
-                
-                return UploadResponse(
-                    document_id=result.get("doc_id", ""),
-                    filename=file.filename,
-                    status="success",
-                    message="Uploaded successfully",
-                    pages=result.get("pages"),
-                    chunks=result.get("chunks")
-                )
-            else:
-                raise HTTPException(400, result.get("message", "Upload failed"))
-    
+        # Save to a temp file so background worker can access it
+        base_dir = os.path.dirname(os.path.abspath(__file__))
+        tmp_dir = os.path.join(base_dir, 'tmp_uploads')
+        os.makedirs(tmp_dir, exist_ok=True)
+        tmp_path = os.path.join(tmp_dir, f"{int(time.time())}_{file.filename}")
+        with open(tmp_path, 'wb') as f:
+            f.write(content)
+
+        # Create persistent job and return immediately
+        job = job_manager.create_job(user_id=user_id, filename=file.filename, file_path=tmp_path)
+
+        return UploadResponse(
+            document_id=job['id'],
+            filename=file.filename,
+            status="processing",
+            message="Upload accepted and is being processed in background"
+        )
+
     except HTTPException:
         raise
     except Exception as e:
-        print(f"❌ Upload error (user {user_id}): {e}")
-        raise HTTPException(500, "Upload failed")
+        print(f"❌ Upload enqueue error (user {user_id}): {e}")
+        raise HTTPException(500, "Failed to enqueue upload")
+
+
+@app.get('/upload-status/{job_id}')
+async def upload_status(job_id: str):
+    job = job_manager.get_job(job_id)
+    if not job:
+        raise HTTPException(404, "Job not found")
+    return job
 
 
 @app.post("/query", response_model=QueryResponse)