feat(retrieval): persist document chunks in PostgreSQL for automatic self-healing across container rebuilds

DocPilot/backend/app/api/documents.py

@@ -11,6 +11,7 @@ from sqlalchemy.orm import Session
 import shutil
 import os
 import uuid
+import json
 
 from DocPilot.backend.app.services.ingestion import (
     process_document,
@@ -148,13 +149,16 @@ async def upload_document(
         db.refresh(document)
 
         try:
-
-            process_document(
+            chunks = process_document(
                 file_path,
                 current_user.id,
                 document.id,
                 mime_type=file.content_type,
             )
+            if chunks:
+                document.chunks_json = json.dumps(chunks)
+                document.chunk_count = len(chunks)
+                db.commit()
 
         except TextExtractionError as exc:
 

DocPilot/backend/app/models/document.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey
+from sqlalchemy import Column, Integer, String, Text, Boolean, DateTime, ForeignKey
 
 from sqlalchemy.sql import func
 
@@ -16,7 +16,6 @@ class Document(Base):
 
     filename = Column(String, nullable=False)
 
-
     filepath = Column(String, nullable=False)
 
     file_size = Column(Integer)
@@ -25,6 +24,8 @@ class Document(Base):
 
     chunk_count = Column(Integer)
 
+    chunks_json = Column(Text, nullable=True)
+
     ocr_used = Column(Boolean, default=False)
 
     status = Column(String, default="processed")

DocPilot/backend/app/services/ingestion.py

@@ -742,3 +742,5 @@ def process_document(
 
     except Exception as e:
         logger.info("[TracePilot] document ingest failed: %r", e)
+
+    return all_chunks

GaugePilot/backend/app/api/documents.py

@@ -10,6 +10,7 @@ from sqlalchemy.orm import Session
 
 import shutil
 import os
+import json
 
 from DocPilot.backend.app.services.ingestion import (
     process_document,
@@ -112,12 +113,16 @@ async def upload_document(
         db.refresh(document)
 
         try:
-            process_document(
+            chunks = process_document(
                 file_path,
                 current_user.id,
                 document.id,
                 mime_type=file.content_type,
             )
+            if chunks:
+                document.chunks_json = json.dumps(chunks)
+                document.chunk_count = len(chunks)
+                db.commit()
         except TextExtractionError as exc:
             db.delete(document)
             db.commit()

pilotcore/retrieval/vector_store.py

@@ -68,9 +68,43 @@ def load_user_documents(user_id: int):
     if os.path.exists(docs_path):
         try:
             with open(docs_path, "rb") as f:
-                return pickle.load(f)
+                docs = pickle.load(f)
+                if docs:
+                    return docs
         except Exception:
-            return []
+            pass
+
+    # Self-Healing Restoration: If container restarted and disk was wiped, restore chunks from database!
+    if user_id:
+        try:
+            import json
+            from DocPilot.backend.app.db.database import SessionLocal
+            from DocPilot.backend.app.models.document import Document
+            db = SessionLocal()
+            user_docs = db.query(Document).filter(Document.owner_id == user_id).all()
+            restored_chunks = []
+            for doc in user_docs:
+                chunks_raw = getattr(doc, "chunks_json", None)
+                if chunks_raw:
+                    try:
+                        parsed = json.loads(chunks_raw)
+                        if isinstance(parsed, list):
+                            restored_chunks.extend(parsed)
+                    except Exception:
+                        pass
+            db.close()
+
+            if restored_chunks:
+                try:
+                    with open(docs_path, "wb") as f:
+                        pickle.dump(restored_chunks, f)
+                    print(f"[Self-Healing] Restored {len(restored_chunks)} document chunks from database for user {user_id}")
+                except Exception:
+                    pass
+                return restored_chunks
+        except Exception as e:
+            print(f"[Self-Healing] Database restoration error: {e}")
+
     return []