feat: add multi-document vectorless RAG, session isolation, and memory services

app/core/dependencies.py

@@ -23,3 +23,10 @@ def get_rag_pipeline() -> VectorlessRAG:
 
 def get_storage() -> MemoryStorage:
     return storage
+
+
+@lru_cache()
+def get_memory_service() -> 'MemoryService':
+    from services.memory_service import MemoryService
+    return MemoryService(get_llm_service())
+

app/core/persistent_db.py

@@ -63,6 +63,40 @@ CREATE TABLE IF NOT EXISTS doc_meta (
     created_at      REAL NOT NULL,
     updated_at      REAL NOT NULL
 );
+
+CREATE TABLE IF NOT EXISTS chat_documents (
+    chat_id     TEXT NOT NULL REFERENCES chats(chat_id) ON DELETE CASCADE,
+    doc_id      TEXT NOT NULL REFERENCES doc_meta(doc_id) ON DELETE CASCADE,
+    doc_name    TEXT,
+    attached_at REAL NOT NULL,
+    PRIMARY KEY (chat_id, doc_id)
+);
+CREATE INDEX IF NOT EXISTS idx_chat_docs ON chat_documents(chat_id);
+
+CREATE TABLE IF NOT EXISTS chat_summaries (
+    chat_id     TEXT PRIMARY KEY REFERENCES chats(chat_id) ON DELETE CASCADE,
+    summary     TEXT NOT NULL,
+    turn_count  INTEGER DEFAULT 0,
+    updated_at  REAL NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS knowledge_memory (
+    doc_id            TEXT PRIMARY KEY REFERENCES doc_meta(doc_id) ON DELETE CASCADE,
+    filename          TEXT NOT NULL,
+    executive_summary TEXT,
+    key_entities      TEXT,
+    key_findings      TEXT,
+    topics            TEXT,
+    updated_at        REAL NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS global_knowledge_summary (
+    id                  TEXT PRIMARY KEY,
+    collective_summary  TEXT NOT NULL,
+    total_documents     INTEGER DEFAULT 0,
+    key_insights        TEXT,
+    updated_at          REAL NOT NULL
+);
 """
 
 
@@ -109,6 +143,13 @@ class PersistentDB:
                 if col not in existing_cols:
                     conn.execute(f"ALTER TABLE doc_meta ADD COLUMN {col} {col_type} DEFAULT {default_val}")
 
+            # Migrate existing chats.doc_id into chat_documents table
+            conn.execute("""
+                INSERT OR IGNORE INTO chat_documents(chat_id, doc_id, doc_name, attached_at)
+                SELECT chat_id, doc_id, doc_name, created_at FROM chats
+                WHERE doc_id IS NOT NULL AND doc_id != ''
+            """)
+
     # ── chat CRUD ───────────────────────────────────────────────────────────
     def create_chat(self, chat_id: Optional[str] = None, title: Optional[str] = None) -> str:
         cid = chat_id or str(uuid.uuid4())
@@ -255,8 +296,201 @@ class PersistentDB:
         with self._tx() as conn:
             cur = conn.execute("DELETE FROM doc_meta WHERE doc_id=?", (doc_id,))
             conn.execute("UPDATE chats SET doc_id=NULL, doc_name=NULL WHERE doc_id=?", (doc_id,))
+            conn.execute("DELETE FROM chat_documents WHERE doc_id=?", (doc_id,))
+            conn.execute("DELETE FROM knowledge_memory WHERE doc_id=?", (doc_id,))
             return cur.rowcount > 0
 
+    # ── multi-doc session associations ──────────────────────────────────────
+    def add_chat_document(self, chat_id: str, doc_id: str, doc_name: Optional[str] = None) -> None:
+        now = time.time()
+        with self._tx() as conn:
+            conn.execute(
+                """
+                INSERT INTO chat_documents(chat_id, doc_id, doc_name, attached_at)
+                VALUES(?,?,?,?)
+                ON CONFLICT(chat_id, doc_id) DO UPDATE SET
+                    doc_name=COALESCE(excluded.doc_name, chat_documents.doc_name),
+                    attached_at=excluded.attached_at
+                """,
+                (chat_id, doc_id, doc_name, now),
+            )
+            # Maintain backward compatibility on chats table with latest doc
+            conn.execute(
+                "UPDATE chats SET doc_id=?, doc_name=?, updated_at=? WHERE chat_id=?",
+                (doc_id, doc_name, now, chat_id),
+            )
+
+    def remove_chat_document(self, chat_id: str, doc_id: str) -> None:
+        now = time.time()
+        with self._tx() as conn:
+            conn.execute("DELETE FROM chat_documents WHERE chat_id=? AND doc_id=?", (chat_id, doc_id))
+            rem = conn.execute(
+                "SELECT doc_id, doc_name FROM chat_documents WHERE chat_id=? ORDER BY attached_at DESC LIMIT 1",
+                (chat_id,),
+            ).fetchone()
+            if rem:
+                conn.execute(
+                    "UPDATE chats SET doc_id=?, doc_name=?, updated_at=? WHERE chat_id=?",
+                    (rem["doc_id"], rem["doc_name"], now, chat_id),
+                )
+            else:
+                conn.execute(
+                    "UPDATE chats SET doc_id=NULL, doc_name=NULL, updated_at=? WHERE chat_id=?",
+                    (now, chat_id),
+                )
+
+    def get_chat_documents(self, chat_id: str) -> List[Dict[str, Any]]:
+        rows = self._conn().execute(
+            """
+            SELECT cd.chat_id, cd.doc_id, cd.doc_name, cd.attached_at,
+                   dm.status, dm.page_count, dm.section_count, dm.filename
+            FROM chat_documents cd
+            LEFT JOIN doc_meta dm ON cd.doc_id = dm.doc_id
+            WHERE cd.chat_id=?
+            ORDER BY cd.attached_at ASC
+            """,
+            (chat_id,),
+        ).fetchall()
+        return [dict(r) for r in rows]
+
+    def clear_chat_documents(self, chat_id: str) -> None:
+        now = time.time()
+        with self._tx() as conn:
+            conn.execute("DELETE FROM chat_documents WHERE chat_id=?", (chat_id,))
+            conn.execute("UPDATE chats SET doc_id=NULL, doc_name=NULL, updated_at=? WHERE chat_id=?", (now, chat_id))
+
+    # ── session long memory & rolling summaries ─────────────────────────────
+    def set_chat_summary(self, chat_id: str, summary: str, turn_count: int = 0) -> None:
+        now = time.time()
+        with self._tx() as conn:
+            conn.execute(
+                """
+                INSERT INTO chat_summaries(chat_id, summary, turn_count, updated_at)
+                VALUES(?,?,?,?)
+                ON CONFLICT(chat_id) DO UPDATE SET
+                    summary=excluded.summary,
+                    turn_count=excluded.turn_count,
+                    updated_at=excluded.updated_at
+                """,
+                (chat_id, summary, turn_count, now),
+            )
+
+    def get_chat_summary(self, chat_id: str) -> Optional[Dict[str, Any]]:
+        row = self._conn().execute(
+            "SELECT * FROM chat_summaries WHERE chat_id=?", (chat_id,)
+        ).fetchone()
+        return dict(row) if row else None
+
+    # ── global knowledge memory & document dossiers ────────────────────────
+    def save_knowledge_dossier(
+        self,
+        doc_id: str,
+        filename: str,
+        executive_summary: str,
+        key_entities: Optional[List[str]] = None,
+        key_findings: Optional[List[str]] = None,
+        topics: Optional[List[str]] = None,
+    ) -> None:
+        now = time.time()
+        with self._tx() as conn:
+            conn.execute(
+                """
+                INSERT INTO knowledge_memory(
+                    doc_id, filename, executive_summary, key_entities, key_findings, topics, updated_at
+                )
+                VALUES(?,?,?,?,?,?,?)
+                ON CONFLICT(doc_id) DO UPDATE SET
+                    filename=excluded.filename,
+                    executive_summary=excluded.executive_summary,
+                    key_entities=excluded.key_entities,
+                    key_findings=excluded.key_findings,
+                    topics=excluded.topics,
+                    updated_at=excluded.updated_at
+                """,
+                (
+                    doc_id,
+                    filename,
+                    executive_summary,
+                    json.dumps(key_entities or []),
+                    json.dumps(key_findings or []),
+                    json.dumps(topics or []),
+                    now,
+                ),
+            )
+
+    def get_knowledge_dossier(self, doc_id: str) -> Optional[Dict[str, Any]]:
+        row = self._conn().execute(
+            "SELECT * FROM knowledge_memory WHERE doc_id=?", (doc_id,)
+        ).fetchone()
+        if not row:
+            return None
+        res = dict(row)
+        for k in ("key_entities", "key_findings", "topics"):
+            if res.get(k) and isinstance(res[k], str):
+                try:
+                    res[k] = json.loads(res[k])
+                except Exception:
+                    res[k] = []
+        return res
+
+    def list_knowledge_dossiers(self) -> List[Dict[str, Any]]:
+        rows = self._conn().execute(
+            "SELECT * FROM knowledge_memory ORDER BY updated_at DESC"
+        ).fetchall()
+        result = []
+        for r in rows:
+            item = dict(r)
+            for k in ("key_entities", "key_findings", "topics"):
+                if item.get(k) and isinstance(item[k], str):
+                    try:
+                        item[k] = json.loads(item[k])
+                    except Exception:
+                        item[k] = []
+            result.append(item)
+        return result
+
+    def delete_knowledge_dossier(self, doc_id: str) -> None:
+        with self._tx() as conn:
+            conn.execute("DELETE FROM knowledge_memory WHERE doc_id=?", (doc_id,))
+
+    def save_global_knowledge_summary(
+        self,
+        collective_summary: str,
+        total_documents: int,
+        key_insights: Optional[List[str]] = None,
+    ) -> None:
+        now = time.time()
+        gid = "global"
+        with self._tx() as conn:
+            conn.execute(
+                """
+                INSERT INTO global_knowledge_summary(
+                    id, collective_summary, total_documents, key_insights, updated_at
+                )
+                VALUES(?,?,?,?,?)
+                ON CONFLICT(id) DO UPDATE SET
+                    collective_summary=excluded.collective_summary,
+                    total_documents=excluded.total_documents,
+                    key_insights=excluded.key_insights,
+                    updated_at=excluded.updated_at
+                """,
+                (gid, collective_summary, total_documents, json.dumps(key_insights or []), now),
+            )
+
+    def get_global_knowledge_summary(self) -> Optional[Dict[str, Any]]:
+        row = self._conn().execute(
+            "SELECT * FROM global_knowledge_summary WHERE id='global'"
+        ).fetchone()
+        if not row:
+            return None
+        res = dict(row)
+        if res.get("key_insights") and isinstance(res["key_insights"], str):
+            try:
+                res["key_insights"] = json.loads(res["key_insights"])
+            except Exception:
+                res["key_insights"] = []
+        return res
+
 
 
 # Global singleton

app/core/rag_pipeline.py

@@ -1,67 +1,148 @@
+import asyncio
 import time
 from typing import Optional, List, Dict, Any
 from utils.logger import app_logger
 
+
 class VectorlessRAG:
     def __init__(self, llm_service):
         self.llm = llm_service
 
-    async def answer_query_stream(
+    async def answer_multi_doc_query_stream(
         self,
         query: str,
-        tree: list,
+        docs: List[Dict[str, Any]],
         chat_history: Optional[List[Dict[str, str]]] = None,
-        doc_name: Optional[str] = None,
-        node_map: Optional[Dict[str, Any]] = None,
-        flat_sections: Optional[List[Dict[str, Any]]] = None,
+        session_summary: Optional[str] = None,
+        global_knowledge: Optional[str] = None,
         deep_analysis: bool = False,
     ):
+        """
+        Executes Vectorless RAG across multiple documents concurrently.
+        Each item in `docs`:
+          {
+            "doc_id": str,
+            "filename": str,
+            "tree": list,
+            "node_map": dict,
+            "flat_sections": list
+          }
+        """
         try:
             t_retrieval_start = time.monotonic()
-            search_result = await self.llm.llm_tree_search(
-                query, tree, chat_history=chat_history, flat_sections=flat_sections
-            )
-            node_ids = search_result.get("node_list", [])
-            nodes = self.llm.find_nodes_by_ids(tree, node_ids, node_map=node_map)
+            all_selected_nodes = []
+
+            # Search across each document tree concurrently
+            async def search_single_doc(doc):
+                tree = doc.get("tree") or []
+                node_map = doc.get("node_map")
+                flat_sections = doc.get("flat_sections")
+                doc_name = doc.get("filename") or "Document"
+
+                search_res = await self.llm.llm_tree_search(
+                    query, tree, chat_history=chat_history, flat_sections=flat_sections
+                )
+                node_ids = search_res.get("node_list", [])
+                nodes = self.llm.find_nodes_by_ids(tree, node_ids, node_map=node_map)
 
-            # If short / referential or overview query returned no explicit node matches, select top overview sections
-            if not nodes:
-                overview_triggers = ["دا", "ده", "دي", "لخص", "ملخص", "اشرح", "بحث", "summary", "summarize", "overview", "what is this", "about"]
-                normalized_q = query.strip().lower()
-                is_short_or_overview = len(normalized_q) <= 8 or any(t in normalized_q for t in overview_triggers)
+                # Fallback to leading sections for overview/comparative queries if no nodes matched
+                if not nodes:
+                    overview_triggers = [
+                        "دا", "ده", "دي", "لخص", "ملخص", "اشرح", "بحث", "مقارنة", "قارن",
+                        "summary", "summarize", "overview", "compare", "comparison", "what is this", "about"
+                    ]
+                    normalized_q = query.strip().lower()
+                    is_short_or_overview = len(normalized_q) <= 10 or any(t in normalized_q for t in overview_triggers)
+                    if is_short_or_overview:
+                        top_nodes = []
+                        def collect_leading(items):
+                            for item in items:
+                                if isinstance(item, dict):
+                                    top_nodes.append(item)
+                                    if len(top_nodes) >= 3:
+                                        return
+                                    if item.get("nodes"):
+                                        collect_leading(item["nodes"])
+                        collect_leading(tree)
+                        nodes = top_nodes
 
-                if is_short_or_overview:
-                    app_logger.info(f"Query '{query}' resolved as overview/referential query with 0 matched nodes. Defaulting to leading document sections.")
-                    top_nodes = []
-                    def collect_leading(items):
-                        for item in items:
-                            if isinstance(item, dict):
-                                top_nodes.append(item)
-                                if len(top_nodes) >= 4:
-                                    return
-                                if item.get("nodes"):
-                                    collect_leading(item["nodes"])
-                    collect_leading(tree)
-                    nodes = top_nodes
+                tagged_nodes = []
+                for n in nodes:
+                    n_copy = dict(n)
+                    n_copy["doc_name"] = doc_name
+                    n_copy["filename"] = doc_name
+                    n_copy["doc_id"] = doc.get("doc_id")
+                    tagged_nodes.append(n_copy)
+                return tagged_nodes
+
+            # Run searches concurrently across all attached documents
+            search_tasks = [search_single_doc(d) for d in docs]
+            results = await asyncio.gather(*search_tasks, return_exceptions=True)
+
+            for r in results:
+                if isinstance(r, list):
+                    all_selected_nodes.extend(r)
+                elif isinstance(r, Exception):
+                    app_logger.error(f"[MULTI-RAG] Document tree search error: {r}")
 
             t_retrieval = time.monotonic() - t_retrieval_start
+            doc_names_list = [d.get("filename", "Doc") for d in docs]
+            doc_names_str = ", ".join(doc_names_list)
+
             app_logger.info(
-                f"[PERF] rag_pipeline retrieval_time={t_retrieval:.2f}s "
-                f"nodes_matched={len(nodes)} deep_analysis={deep_analysis}"
+                f"[PERF] multi_rag retrieval_time={t_retrieval:.2f}s "
+                f"docs_count={len(docs)} nodes_matched={len(all_selected_nodes)} deep_analysis={deep_analysis}"
             )
 
-            # Signal retrieval completion to frontend immediately
-            yield f'<!--STAGE:{{"stage":"retrieved","duration":{t_retrieval:.2f},"nodes":{len(nodes)}}}-->\n'
+            # Signal retrieval completion stage to frontend
+            yield f'<!--STAGE:{{"stage":"retrieved","duration":{t_retrieval:.2f},"nodes":{len(all_selected_nodes)},"docs":{len(docs)}}}-->\n'
 
+            # Stream final synthesis with multi-document attribution
             async for chunk in self.llm.generate_answer_stream(
-                query, nodes, chat_history=chat_history, doc_name=doc_name, deep_analysis=deep_analysis
+                query=query,
+                nodes=all_selected_nodes,
+                chat_history=chat_history,
+                doc_name=doc_names_str,
+                session_summary=session_summary,
+                global_knowledge=global_knowledge,
+                deep_analysis=deep_analysis,
             ):
                 yield chunk
 
         except Exception as e:
-            app_logger.error(f"Error in RAG pipeline: {str(e)}", exc_info=True)
+            app_logger.error(f"Error in multi-doc RAG pipeline: {str(e)}", exc_info=True)
             is_arabic = any("\u0600" <= c <= "\u06ff" for c in query)
             if is_arabic:
-                yield "حدث خطأ أثناء استرجاع المعلومات من المستند المفهرس. يرجى إعادة المحاولة."
+                yield "حدث خطأ أثناء استرجاع المعلومات من المستندات المفهرسة. يرجى إعادة المحاولة."
             else:
-                yield "An error occurred while retrieving information from the indexed document. Please try again."
+                yield "An error occurred while retrieving information from the indexed documents. Please try again."
+
+    async def answer_query_stream(
+        self,
+        query: str,
+        tree: list,
+        chat_history: Optional[List[Dict[str, str]]] = None,
+        doc_name: Optional[str] = None,
+        node_map: Optional[Dict[str, Any]] = None,
+        flat_sections: Optional[List[Dict[str, Any]]] = None,
+        deep_analysis: bool = False,
+        session_summary: Optional[str] = None,
+        global_knowledge: Optional[str] = None,
+    ):
+        """Single-document backward-compatible wrapper."""
+        doc_obj = {
+            "doc_id": "single",
+            "filename": doc_name or "Uploaded Document",
+            "tree": tree,
+            "node_map": node_map,
+            "flat_sections": flat_sections,
+        }
+        async for chunk in self.answer_multi_doc_query_stream(
+            query=query,
+            docs=[doc_obj],
+            chat_history=chat_history,
+            session_summary=session_summary,
+            global_knowledge=global_knowledge,
+            deep_analysis=deep_analysis,
+        ):
+            yield chunk

app/core/storage.py

@@ -485,13 +485,35 @@ class MemoryStorage:
             doc = self._documents.get(doc_id)
             doc_name = doc.get("filename") if doc else None
         persistent_db.set_chat_doc(chat_id=chat_id, doc_id=doc_id, doc_name=doc_name)
+        persistent_db.add_chat_document(chat_id=chat_id, doc_id=doc_id, doc_name=doc_name)
+
+    def add_chat_doc(self, chat_id: str, doc_id: str, doc_name: Optional[str] = None) -> None:
+        if not self.chat_exists(chat_id):
+            persistent_db.create_chat(chat_id=chat_id)
+        if doc_name is None:
+            doc = self._documents.get(doc_id)
+            doc_name = doc.get("filename") if doc else None
+        persistent_db.add_chat_document(chat_id=chat_id, doc_id=doc_id, doc_name=doc_name)
 
     def get_chat_doc(self, chat_id: str) -> Optional[str]:
+        # Return primary attached doc or first multi-doc
         meta = persistent_db.get_chat(chat_id)
-        return meta["doc_id"] if meta else None
+        if meta and meta.get("doc_id"):
+            return meta["doc_id"]
+        docs = persistent_db.get_chat_documents(chat_id)
+        return docs[0]["doc_id"] if docs else None
+
+    def get_chat_docs(self, chat_id: str) -> List[Dict[str, Any]]:
+        return persistent_db.get_chat_documents(chat_id)
 
-    def remove_chat_doc(self, chat_id: str) -> None:
-        persistent_db.remove_chat_doc(chat_id)
+    def remove_chat_doc(self, chat_id: str, doc_id: Optional[str] = None) -> None:
+        if doc_id:
+            persistent_db.remove_chat_document(chat_id, doc_id)
+        else:
+            persistent_db.remove_chat_doc(chat_id)
+
+    def clear_chat_docs(self, chat_id: str) -> None:
+        persistent_db.clear_chat_documents(chat_id)
 
     def delete_chat(self, chat_id: str) -> None:
         persistent_db.delete_chat(chat_id)
@@ -502,6 +524,56 @@ class MemoryStorage:
     def update_chat_title(self, chat_id: str, title: str) -> None:
         persistent_db.update_chat_title(chat_id=chat_id, title=title)
 
+    # ── session long memory & rolling summaries ─────────────────────────────
+    def set_chat_summary(self, chat_id: str, summary: str, turn_count: int = 0) -> None:
+        persistent_db.set_chat_summary(chat_id=chat_id, summary=summary, turn_count=turn_count)
+
+    def get_chat_summary(self, chat_id: str) -> Optional[Dict[str, Any]]:
+        return persistent_db.get_chat_summary(chat_id=chat_id)
+
+    # ── global knowledge memory & document dossiers ────────────────────────
+    def save_knowledge_dossier(
+        self,
+        doc_id: str,
+        filename: str,
+        executive_summary: str,
+        key_entities: Optional[List[str]] = None,
+        key_findings: Optional[List[str]] = None,
+        topics: Optional[List[str]] = None,
+    ) -> None:
+        persistent_db.save_knowledge_dossier(
+            doc_id=doc_id,
+            filename=filename,
+            executive_summary=executive_summary,
+            key_entities=key_entities,
+            key_findings=key_findings,
+            topics=topics,
+        )
+
+    def get_knowledge_dossier(self, doc_id: str) -> Optional[Dict[str, Any]]:
+        return persistent_db.get_knowledge_dossier(doc_id)
+
+    def list_knowledge_dossiers(self) -> List[Dict[str, Any]]:
+        return persistent_db.list_knowledge_dossiers()
+
+    def delete_knowledge_dossier(self, doc_id: str) -> None:
+        persistent_db.delete_knowledge_dossier(doc_id)
+
+    def save_global_knowledge_summary(
+        self,
+        collective_summary: str,
+        total_documents: int,
+        key_insights: Optional[List[str]] = None,
+    ) -> None:
+        persistent_db.save_global_knowledge_summary(
+            collective_summary=collective_summary,
+            total_documents=total_documents,
+            key_insights=key_insights,
+        )
+
+    def get_global_knowledge_summary(self) -> Optional[Dict[str, Any]]:
+        return persistent_db.get_global_knowledge_summary()
+
 
 # Global singleton storage instance
 storage = MemoryStorage()

app/routers/__init__.py

@@ -1,9 +1,11 @@
 from fastapi import APIRouter
 from routers.documents import router as documents_router
 from routers.chat import router as chat_router
+from routers.memory import router as memory_router
 
 api_router = APIRouter()
 api_router.include_router(documents_router)
 api_router.include_router(chat_router)
+api_router.include_router(memory_router)
 
-__all__ = ["api_router", "documents_router", "chat_router"]
+__all__ = ["api_router", "documents_router", "chat_router", "memory_router"]

app/routers/chat.py

@@ -1,22 +1,32 @@
+import asyncio
+import re
+from typing import List, Optional
 from fastapi import APIRouter, HTTPException, Depends
 from fastapi.responses import StreamingResponse
 from pydantic import BaseModel
 
-from schemas.schemas import AskRequest, NewChatResponse, ChatHistoryResponse
+from schemas.schemas import (
+    AskRequest,
+    NewChatResponse,
+    ChatHistoryResponse,
+    AttachDocRequest,
+    AttachMultipleDocsRequest,
+)
 from core.storage import MemoryStorage
-from core.dependencies import get_llm_service, get_rag_pipeline, get_storage
+from core.dependencies import (
+    get_llm_service,
+    get_rag_pipeline,
+    get_storage,
+    get_memory_service,
+)
 from core.rag_pipeline import VectorlessRAG
 from services.llm_service import LLMService
+from services.memory_service import MemoryService
 from utils.logger import app_logger
-import re
 
 router = APIRouter(tags=["Chat"])
 
 
-class AttachDocRequest(BaseModel):
-    doc_id: str
-
-
 @router.post("/chat/new", response_model=NewChatResponse)
 async def create_chat(storage: MemoryStorage = Depends(get_storage)):
     chat_id = storage.create_chat()
@@ -30,23 +40,75 @@ async def attach_document_to_chat(
     body: AttachDocRequest,
     storage: MemoryStorage = Depends(get_storage),
 ):
-    """Attach an indexed document to an active conversation."""
+    """Attach an indexed document to an active conversation session."""
     if not storage.tree_exists(body.doc_id):
         raise HTTPException(status_code=404, detail="Document ID not found in memory")
-    storage.set_chat_doc(chat_id, body.doc_id)
+    storage.add_chat_doc(chat_id, body.doc_id, doc_name=body.doc_name)
     app_logger.info(f"Attached document {body.doc_id} to chat session {chat_id}")
     return {"message": "Document attached successfully", "chat_id": chat_id, "doc_id": body.doc_id}
 
 
+@router.post("/chat/{chat_id}/attach-multiple")
+async def attach_multiple_documents_to_chat(
+    chat_id: str,
+    body: AttachMultipleDocsRequest,
+    storage: MemoryStorage = Depends(get_storage),
+):
+    """Attach multiple indexed documents to an active conversation session."""
+    attached = []
+    for doc_id in body.doc_ids:
+        if storage.tree_exists(doc_id):
+            storage.add_chat_doc(chat_id, doc_id)
+            attached.append(doc_id)
+    app_logger.info(f"Attached {len(attached)} documents to chat session {chat_id}")
+    return {"message": f"Attached {len(attached)} documents successfully", "chat_id": chat_id, "doc_ids": attached}
+
+
+@router.get("/chat/{chat_id}/documents")
+async def get_chat_documents(
+    chat_id: str,
+    storage: MemoryStorage = Depends(get_storage),
+):
+    """Return all attached documents for this session with status and page counts."""
+    docs = storage.get_chat_docs(chat_id)
+    detailed_docs = []
+    for d in docs:
+        doc_info = storage.get_document(d["doc_id"])
+        if doc_info:
+            detailed_docs.append({
+                "doc_id": d["doc_id"],
+                "filename": doc_info.get("filename", d.get("doc_name", "document.pdf")),
+                "status": doc_info.get("status", "ready"),
+                "page_count": doc_info.get("page_count", 1),
+                "section_count": doc_info.get("section_count", 0),
+            })
+        else:
+            detailed_docs.append(d)
+    return {"chat_id": chat_id, "documents": detailed_docs}
+
+
+@router.delete("/chat/{chat_id}/document/{doc_id}")
+async def detach_specific_document_from_chat(
+    chat_id: str,
+    doc_id: str,
+    storage: MemoryStorage = Depends(get_storage),
+):
+    """Detach a specific document from the active chat session."""
+    storage.remove_chat_doc(chat_id, doc_id=doc_id)
+    app_logger.info(f"Detached document {doc_id} from chat session {chat_id}")
+    return {"message": "Document detached successfully", "chat_id": chat_id, "doc_id": doc_id}
+
+
 @router.delete("/chat/{chat_id}/document")
 async def detach_document_from_chat(
     chat_id: str,
     storage: MemoryStorage = Depends(get_storage),
 ):
-    """Detach document from active chat session."""
+    """Detach all documents from active chat session."""
+    storage.clear_chat_docs(chat_id)
     storage.remove_chat_doc(chat_id)
-    app_logger.info(f"Detached document from chat session {chat_id}")
-    return {"message": "Document detached successfully", "chat_id": chat_id}
+    app_logger.info(f"Cleared all documents from chat session {chat_id}")
+    return {"message": "All documents detached successfully", "chat_id": chat_id}
 
 
 @router.post("/ask")
@@ -55,96 +117,134 @@ async def ask_question(
     storage: MemoryStorage = Depends(get_storage),
     llm_service: LLMService = Depends(get_llm_service),
     rag_pipeline: VectorlessRAG = Depends(get_rag_pipeline),
+    memory_service: MemoryService = Depends(get_memory_service),
 ):
-    chat_history = storage.ensure_chat(request.chat_id)
-
-    # 1. Resolve active document identity for this chat session
-    effective_doc_id = request.doc_id or storage.get_chat_doc(request.chat_id)
-    doc_state = storage.get_document(effective_doc_id) if effective_doc_id else None
-
-    # If request passed doc_id, ensure session association is persisted
-    if request.doc_id and effective_doc_id:
-        storage.set_chat_doc(request.chat_id, effective_doc_id)
-
-    # 2. Deterministic routing decision
-    error_detail = None
-    if not effective_doc_id:
-        selected_mode = "conversational"
-    elif doc_state is None:
-        selected_mode = "error"
-        error_detail = "Document session not found or expired"
-    elif doc_state.get("status") == "indexing":
-        selected_mode = "indexing"
-    elif doc_state.get("status") == "ready" and doc_state.get("tree"):
-        selected_mode = "pageindex"
+    # Ensure chat session exists in persistent storage
+    storage.ensure_chat(request.chat_id)
+
+    # 1. Resolve target document IDs for this query session
+    target_doc_ids: List[str] = []
+    if request.doc_ids:
+        target_doc_ids.extend(request.doc_ids)
+    elif request.doc_id:
+        target_doc_ids.append(request.doc_id)
+
+    if target_doc_ids:
+        # Associate newly requested docs with this session
+        for did in target_doc_ids:
+            storage.add_chat_doc(request.chat_id, did)
     else:
+        # Check existing docs attached to this session
+        session_docs = storage.get_chat_docs(request.chat_id)
+        if session_docs:
+            target_doc_ids = [d["doc_id"] for d in session_docs]
+        else:
+            primary_id = storage.get_chat_doc(request.chat_id)
+            if primary_id:
+                target_doc_ids = [primary_id]
+
+    # Deduplicate while preserving order
+    target_doc_ids = list(dict.fromkeys(target_doc_ids))
+
+    # 2. Inspect document states
+    ready_docs = []
+    indexing_docs = []
+    error_docs = []
+
+    for did in target_doc_ids:
+        doc_state = storage.get_document(did)
+        if not doc_state:
+            error_docs.append({"doc_id": did, "filename": did, "error": "Document not found or expired"})
+        elif doc_state.get("status") == "ready" and doc_state.get("tree"):
+            node_map = storage.get_node_map(did)
+            flat_sections = storage.get_flat_sections(did)
+            if node_map is None or flat_sections is None:
+                node_map, flat_sections = llm_service.build_index_maps(doc_state["tree"])
+                storage.set_node_map(did, node_map)
+                storage.set_flat_sections(did, flat_sections)
+            ready_docs.append({
+                "doc_id": did,
+                "filename": doc_state.get("filename", "document.pdf"),
+                "tree": doc_state["tree"],
+                "node_map": node_map,
+                "flat_sections": flat_sections,
+            })
+        elif doc_state.get("status") == "indexing":
+            indexing_docs.append(doc_state)
+        else:
+            error_docs.append(doc_state)
+
+    # 3. Determine execution mode
+    if ready_docs:
+        selected_mode = "multi_doc_pageindex"
+    elif indexing_docs:
+        selected_mode = "indexing"
+    elif error_docs and not ready_docs:
         selected_mode = "error"
-        error_detail = doc_state.get("error") or "Document tree unavailable"
+    else:
+        # Pure conversational AI assistant with workspace global knowledge
+        selected_mode = "conversational"
 
     # Router Audit Logging
     app_logger.info(
-        f"[CHAT ROUTER] conversation_id={request.chat_id} document_id={effective_doc_id} "
-        f"document_status={doc_state.get('status') if doc_state else 'none'} "
-        f"tree_ready={bool(doc_state and doc_state.get('tree'))} mode={selected_mode} "
-        f"deep_analysis={bool(request.deep_analysis)}"
+        f"[CHAT ROUTER] conversation_id={request.chat_id} target_docs={len(target_doc_ids)} "
+        f"ready_docs={len(ready_docs)} indexing_docs={len(indexing_docs)} "
+        f"mode={selected_mode} deep_analysis={bool(request.deep_analysis)}"
     )
 
-    # Use concise recent history directly to preserve TTFT without blocking LLM calls
-    effective_history = chat_history[-6:] if len(chat_history) > 6 else chat_history
+    # 4. Retrieve session memory (isolated short-term window + long-term rolling summary)
+    mem_context = memory_service.get_session_context(request.chat_id)
+    recent_turns = mem_context.get("recent_turns", [])
+    session_summary = mem_context.get("session_summary")
+
+    # 5. Retrieve global workspace knowledge summary
+    global_knowledge = memory_service.get_global_knowledge()
 
     is_arabic = any("\u0600" <= c <= "\u06ff" for c in request.query)
-    doc_name = (doc_state.get("filename") if doc_state else None) or request.doc_name
 
     async def stream_generator():
         full_answer = ""
         try:
-            if selected_mode == "pageindex":
-                # Mode A: Deterministic PageIndex tree retrieval & grounded synthesis
-                tree = doc_state["tree"]
-                node_map = storage.get_node_map(effective_doc_id)
-                flat_sections = storage.get_flat_sections(effective_doc_id)
-                if node_map is None or flat_sections is None:
-                    node_map, flat_sections = llm_service.build_index_maps(tree)
-                    storage.set_node_map(effective_doc_id, node_map)
-                    storage.set_flat_sections(effective_doc_id, flat_sections)
-
-                async for chunk in rag_pipeline.answer_query_stream(
-                    request.query,
-                    tree,
-                    chat_history=effective_history,
-                    doc_name=doc_name,
-                    node_map=node_map,
-                    flat_sections=flat_sections,
+            if selected_mode == "multi_doc_pageindex":
+                # Mode A: Deterministic Vectorless multi-doc tree retrieval & grounded synthesis
+                async for chunk in rag_pipeline.answer_multi_doc_query_stream(
+                    query=request.query,
+                    docs=ready_docs,
+                    chat_history=recent_turns,
+                    session_summary=session_summary,
+                    global_knowledge=global_knowledge,
                     deep_analysis=bool(request.deep_analysis),
                 ):
                     full_answer += chunk
                     yield chunk
 
             elif selected_mode == "indexing":
-                # Mode B: Document attached but still indexing - inform user explicitly
-                filename_display = doc_name or "PDF"
+                # Mode B: Document(s) attached but still indexing
+                doc_names = ", ".join([d.get("filename", "PDF") for d in indexing_docs])
                 if is_arabic:
-                    msg = f"المستند ({filename_display}) قيد المعالجة والفهرسة حالياً. يرجى الانتظار لحظة حتى تكتمل الفهرسة."
+                    msg = f"المستندات التالية قيد الفهرسة حالياً ({doc_names}). يرجى الانتظار لحظة حتى تكتمل المعالجة."
                 else:
-                    msg = f"Your document ({filename_display}) is currently being indexed. Please wait a moment."
+                    msg = f"The following documents are currently being indexed ({doc_names}). Please wait a moment."
                 full_answer = msg
                 yield msg
 
             elif selected_mode == "error":
-                # Mode C: Document failed or unavailable - do NOT fall back to conversational
-                filename_display = doc_name or "PDF"
-                detail_str = error_detail or "File unavailable"
+                # Mode C: Requested documents failed or unavailable
+                err_names = ", ".join([d.get("filename", "PDF") for d in error_docs])
                 if is_arabic:
-                    msg = f"تعذر الوصول إلى المستند المطلوب ({detail_str}). يرجى إعادة رفع الملف."
+                    msg = f"تعذر استرجاع المستندات التالية ({err_names}). يرجى إعادة رفع الملفات."
                 else:
-                    msg = f"Could not access the requested document ({detail_str}). Please try re-uploading the file."
+                    msg = f"Could not access the requested documents ({err_names}). Please try re-uploading."
                 full_answer = msg
                 yield msg
 
             else:
-                # Mode D: Pure conversational AI assistant (strictly when no document is attached)
+                # Mode D: Pure conversational AI assistant with workspace collective knowledge injected
                 async for chunk in llm_service.generate_conversational_stream(
-                    request.query, chat_history
+                    query=request.query,
+                    chat_history=recent_turns,
+                    session_summary=session_summary,
+                    global_knowledge=global_knowledge,
                 ):
                     full_answer += chunk
                     yield chunk
@@ -156,6 +256,13 @@ async def ask_question(
                 storage.add_message(
                     request.chat_id, role="assistant", content=clean_answer or full_answer
                 )
+                # Asynchronously update long-term rolling summary if conversation threshold reached
+                try:
+                    asyncio.create_task(
+                        memory_service.update_session_summary_if_needed(request.chat_id)
+                    )
+                except Exception:
+                    pass
 
     return StreamingResponse(stream_generator(), media_type="text/event-stream")
 
@@ -171,7 +278,7 @@ async def get_chat_history(
     return ChatHistoryResponse(history=history)
 
 
-# ── Long Memory Endpoints ─────────────────────────────────────────────────
+# ── Long Memory & Conversation Management Endpoints ────────────────────────
 
 @router.get("/chats")
 async def list_all_chats(storage: MemoryStorage = Depends(get_storage)):

app/routers/documents.py

@@ -75,6 +75,21 @@ async def run_background_indexing(
         )
         storage.set_hash_mapping(content_hash, doc_id)
 
+        # Trigger background knowledge dossier generation and global synthesis
+        try:
+            from core.dependencies import get_memory_service
+            mem_service = get_memory_service()
+            asyncio.create_task(
+                mem_service.generate_document_dossier(
+                    doc_id=doc_id,
+                    filename=filename,
+                    tree=tree,
+                    flat_sections=flat_sections,
+                )
+            )
+        except Exception as mex:
+            app_logger.warning(f"Failed to launch dossier generation for {doc_id}: {mex}")
+
         app_logger.info(
             f"[INDEX] document_id={doc_id} stage=ready duration={total_duration:.2f}s "
             f"method={indexing_method} pages={page_count} sections={section_count}"

app/routers/memory.py

@@ -0,0 +1,98 @@
+import time
+from fastapi import APIRouter, HTTPException, Depends
+from typing import Optional, List, Dict, Any
+from pydantic import BaseModel
+
+from core.dependencies import get_memory_service, get_storage
+from core.storage import MemoryStorage
+from services.memory_service import MemoryService
+from utils.logger import app_logger
+
+router = APIRouter(prefix="/memory", tags=["Memory & Knowledge"])
+
+
+@router.get("/knowledge")
+async def get_global_knowledge(
+    memory_service: MemoryService = Depends(get_memory_service),
+):
+    """
+    Retrieve workspace collective knowledge summary and all per-document knowledge dossiers.
+    Provides cross-document intelligence and long-term memory across sessions.
+    """
+    return memory_service.get_global_knowledge()
+
+
+@router.post("/knowledge/refresh")
+async def refresh_global_knowledge(
+    memory_service: MemoryService = Depends(get_memory_service),
+):
+    """
+    Synthesize and rebuild the global workspace knowledge memory across all indexed documents.
+    """
+    app_logger.info("Triggered manual refresh of global knowledge memory")
+    result = await memory_service.synthesize_global_knowledge()
+    dossiers = memory_service.list_knowledge_dossiers()
+    return {
+        "message": "Global knowledge memory refreshed successfully",
+        "collective_summary": result.get("collective_summary"),
+        "key_insights": result.get("key_insights", []),
+        "total_documents": result.get("total_documents", len(dossiers)),
+        "dossiers": dossiers,
+    }
+
+
+@router.get("/dossier/{doc_id}")
+async def get_document_dossier(
+    doc_id: str,
+    memory_service: MemoryService = Depends(get_memory_service),
+):
+    """
+    Retrieve structured knowledge dossier for an individual document.
+    """
+    dossier = memory_service.get_document_dossier(doc_id)
+    if not dossier:
+        raise HTTPException(status_code=404, detail="Knowledge dossier not found for this document")
+    return dossier
+
+
+@router.get("/chat/{chat_id}")
+async def get_chat_memory_context(
+    chat_id: str,
+    memory_service: MemoryService = Depends(get_memory_service),
+    storage: MemoryStorage = Depends(get_storage),
+):
+    """
+    Inspect the short-term and long-term isolated memory for a specific chat session.
+    """
+    if not storage.chat_exists(chat_id):
+        raise HTTPException(status_code=404, detail="Chat session not found")
+
+    context = memory_service.get_session_context(chat_id)
+    attached_docs = storage.get_chat_docs(chat_id)
+    return {
+        "chat_id": chat_id,
+        "session_summary": context.get("session_summary"),
+        "recent_turns_count": len(context.get("recent_turns", [])),
+        "recent_turns": context.get("recent_turns", []),
+        "attached_documents": attached_docs,
+    }
+
+
+@router.post("/chat/{chat_id}/summarize")
+async def force_summarize_chat(
+    chat_id: str,
+    memory_service: MemoryService = Depends(get_memory_service),
+    storage: MemoryStorage = Depends(get_storage),
+):
+    """
+    Force-generate or update the rolling long-term memory summary for this chat session.
+    """
+    if not storage.chat_exists(chat_id):
+        raise HTTPException(status_code=404, detail="Chat session not found")
+
+    summary = await memory_service.update_session_summary_if_needed(chat_id, force=True)
+    return {
+        "message": "Chat session summary updated",
+        "chat_id": chat_id,
+        "summary": summary,
+    }

app/schemas/schemas.py

@@ -5,9 +5,17 @@ class AskRequest(BaseModel):
     chat_id: str = Field(..., description="Active chat session ID")
     query: str = Field(..., description="User query / question")
     doc_id: Optional[str] = Field(None, description="Optional document ID if attached")
+    doc_ids: Optional[List[str]] = Field(None, description="Optional list of document IDs for multi-doc search")
     doc_name: Optional[str] = Field(None, description="Optional document name if known")
     deep_analysis: Optional[bool] = Field(False, description="Enable deep analysis mode")
 
+class AttachDocRequest(BaseModel):
+    doc_id: str
+    doc_name: Optional[str] = None
+
+class AttachMultipleDocsRequest(BaseModel):
+    doc_ids: List[str]
+
 class UploadResponse(BaseModel):
     message: str = "Success"
     doc_id: str

app/services/llm_service.py

@@ -296,6 +296,8 @@ CONVERSATION HISTORY
 
         return ""
 
+    summarize_conversation = summarize_history
+
     async def llm_tree_search(
         self,
         query: str,
@@ -693,6 +695,8 @@ If there is insufficient evidence:
         chat_history: list = None,
         doc_name: Optional[str] = None,
         model: Optional[str] = None,
+        session_summary: Optional[str] = None,
+        global_knowledge: Optional[str] = None,
         deep_analysis: bool = False,
     ):
         """Chat & synthesis model generates streaming answer with multimodal citations."""
@@ -717,10 +721,13 @@ If there is insufficient evidence:
                 text = (node.get("text") or node.get("content") or "Content not available.").strip()
                 if len(text) > max_node_chars:
                     text = text[:max_node_chars].rstrip() + "\n[...section continues...]"
-                part = f"[Section: '{title}' | Page {page}]\n{text}{image_markers}"
+                
+                doc_label = node.get("doc_name") or node.get("filename")
+                doc_prefix = f"Document: '{doc_label}' | " if doc_label else ""
+                part = f"[{doc_prefix}Section: '{title}' | Page {page}]\n{text}{image_markers}"
                 if current_total_chars + len(part) > max_total_chars and context_parts:
                     remaining_budget = max(500, max_total_chars - current_total_chars)
-                    part = f"[Section: '{title}' | Page {page}]\n{text[:remaining_budget].rstrip()}\n[...truncated for context limit...]{image_markers}"
+                    part = f"[{doc_prefix}Section: '{title}' | Page {page}]\n{text[:remaining_budget].rstrip()}\n[...truncated for context limit...]{image_markers}"
                     context_parts.append(part)
                     break
                 context_parts.append(part)
@@ -730,6 +737,14 @@ If there is insufficient evidence:
 
         effective_doc_name = doc_name or "Uploaded Document"
 
+        session_summary_str = ""
+        if session_summary:
+            session_summary_str = f"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nSESSION LONG-TERM MEMORY\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n{session_summary}\n\n"
+
+        global_knowledge_str = ""
+        if global_knowledge:
+            global_knowledge_str = f"━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nGLOBAL KNOWLEDGE MEMORY (All workspace documents)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n{global_knowledge}\n\n"
+
         system_prompt = f"""You are Atlas, a production-grade document intelligence assistant operating inside a Retrieval-Augmented Generation (RAG) system.
 Your name is Atlas. When asked about your identity, name, or purpose, introduce yourself clearly as Atlas, the Document Intelligence research assistant.
 
@@ -1041,7 +1056,7 @@ CHAT HISTORY = CONTEXT
 
 Never treat an assumption from chat history as a document fact.
 
-━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+{session_summary_str}{global_knowledge_str}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 DOCUMENT CONTEXT
 ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
@@ -1168,9 +1183,12 @@ Return only the final user-facing answer."""
         query: str,
         chat_history: list = None,
         model: Optional[str] = None,
+        session_summary: Optional[str] = None,
+        global_knowledge: Optional[str] = None,
     ):
-        """Conversational stream when no document is attached yet.
+        """Conversational stream when no specific document is attached yet.
         Uses the smaller/faster retrieval model for low-latency chat, with automatic Groq failover.
+        Integrates Session Long-Term Memory and Global Knowledge Memory across all documents.
         """
         # Use chat_model (Qwen2.5-7B from HF_CHAT_MODEL env) — fast and capable for conversational use
         target_model = model or self.chat_model
@@ -1192,6 +1210,12 @@ Always match the user's language.
 Be concise and accurate. Never invent facts.
 If the user asks about a document and none is uploaded, tell them to upload a PDF.
 """
+        if session_summary:
+            system_prompt += f"\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\nSESSION LONG-TERM MEMORY (Summary of earlier turns)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n{session_summary}\n"
+
+        if global_knowledge:
+            system_prompt += f"\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━���━━━\nGLOBAL KNOWLEDGE MEMORY (All workspace documents)\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n{global_knowledge}\nYou can answer questions about the collective documents in the workspace using the Global Knowledge Memory above.\n"
+
         if chat_history_text:
             system_prompt += f"\n[Recent conversation]\n{chat_history_text}\n"
 

app/services/memory_service.py

@@ -0,0 +1,271 @@
+"""
+memory_service.py
+─────────────────
+Manages Multi-Tier Memory for Atlas Document Intelligence:
+1. Short-Term Session Memory: Strict per-session sliding context window.
+2. Long-Term Session Memory: Rolling conversation summaries for long chats.
+3. Document Knowledge Dossiers: Synthesized core findings & entities per document.
+4. Global Knowledge Memory: Collective synthesis across all ingested documents.
+"""
+
+import json
+import time
+from typing import Any, Dict, List, Optional
+
+from core.storage import storage
+from utils.logger import app_logger
+
+
+class MemoryService:
+    def __init__(self, llm_service) -> None:
+        self.llm = llm_service
+
+    # ── 1. Short-Term & Long-Term Session Memory ────────────────────────────
+    def get_session_context(
+        self, chat_id: str, max_recent_turns: int = 6
+    ) -> Dict[str, Any]:
+        """
+        Retrieves strictly isolated session memory:
+        - Recent turns (short-term memory window)
+        - Rolling session summary (long-term memory)
+        """
+        full_history = storage.get_chat_history(chat_id) or []
+        summary_record = storage.get_chat_summary(chat_id)
+        session_summary = summary_record.get("summary") if summary_record else None
+
+        recent_turns = full_history[-max_recent_turns:] if len(full_history) > max_recent_turns else full_history
+
+        return {
+            "chat_id": chat_id,
+            "session_summary": session_summary,
+            "recent_turns": recent_turns,
+            "total_messages": len(full_history),
+        }
+
+    async def update_session_summary_if_needed(self, chat_id: str) -> None:
+        """Trigger rolling summarization if conversation length exceeds threshold."""
+        try:
+            full_history = storage.get_chat_history(chat_id) or []
+            if len(full_history) < 8:
+                return
+
+            last_summary = storage.get_chat_summary(chat_id)
+            last_count = last_summary.get("turn_count", 0) if last_summary else 0
+
+            # Only re-summarize if at least 4 new messages occurred
+            if (len(full_history) - last_count) < 4:
+                return
+
+            app_logger.info(f"[MEMORY] Updating rolling summary for chat_id={chat_id}")
+            summary = await self.llm.summarize_conversation(full_history)
+            if summary:
+                storage.set_chat_summary(chat_id, summary, turn_count=len(full_history))
+                app_logger.info(f"[MEMORY] Successfully updated summary for chat_id={chat_id}")
+        except Exception as e:
+            app_logger.error(f"[MEMORY] Error updating session summary: {e}")
+
+    # ── 2. Document Knowledge Dossier ──────────────────────────────────────
+    async def generate_document_dossier(
+        self, doc_id: str, filename: str, tree: list, flat_sections: Optional[list] = None
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Analyzes a newly indexed document to generate an executive dossier:
+        - Executive summary
+        - Key entities
+        - Key findings
+        - Topics
+        """
+        try:
+            # Build representative text preview from first few sections or flat sections
+            sections_to_read = flat_sections[:8] if flat_sections else []
+            preview_texts = []
+            for s in sections_to_read:
+                title = s.get("title", "Section")
+                text = (s.get("text") or "")[:800]
+                preview_texts.append(f"### {title}\n{text}")
+
+            if not preview_texts:
+                # Fallback to tree node text
+                for node in tree[:5]:
+                    if isinstance(node, dict):
+                        t = node.get("title", "")
+                        c = (node.get("text") or "")[:800]
+                        preview_texts.append(f"### {t}\n{c}")
+
+            combined_preview = "\n\n".join(preview_texts)[:4000]
+            if not combined_preview.strip():
+                combined_preview = f"Document: {filename} with {len(tree)} outline nodes."
+
+            prompt = f"""You are the Document Intelligence Synthesizer for Atlas.
+Analyze the following document preview and extract a structured knowledge dossier.
+
+DOCUMENT: {filename}
+CONTENT PREVIEW:
+{combined_preview}
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+OUTPUT REQUIREMENTS
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Return ONLY valid JSON matching this schema:
+{{
+  "executive_summary": "A 2-3 paragraph comprehensive summary of the document purpose, findings, and themes.",
+  "key_entities": ["Entity 1", "Entity 2", "Entity 3"],
+  "key_findings": ["Key finding 1", "Key finding 2", "Key finding 3"],
+  "topics": ["Topic 1", "Topic 2", "Topic 3", "Topic 4"]
+}}
+"""
+            raw_result = ""
+            if not self.llm.hf_exhausted:
+                try:
+                    resp = await self.llm.client.chat_completion(
+                        messages=[{"role": "user", "content": prompt}],
+                        model=self.llm.chat_model,
+                        temperature=0.2,
+                        max_tokens=800,
+                    )
+                    raw_result = resp.choices[0].message.content or ""
+                except Exception as e:
+                    self.llm._is_hf_exhausted(e)
+                    app_logger.warning(f"HF dossier generation failed ({e}). Checking Groq fallback...")
+
+            if (not raw_result or self.llm.hf_exhausted) and self.llm.groq_client:
+                groq_resp = await self.llm.groq_client.chat.completions.create(
+                    model=self.llm.groq_chat_model,
+                    messages=[{"role": "user", "content": prompt}],
+                    temperature=0.2,
+                    max_tokens=800,
+                )
+                raw_result = groq_resp.choices[0].message.content or ""
+
+            data = self.llm._extract_json(raw_result)
+            summary = data.get("executive_summary") or f"Knowledge dossier for {filename}."
+            entities = data.get("key_entities") or []
+            findings = data.get("key_findings") or []
+            topics = data.get("topics") or []
+
+            storage.save_knowledge_dossier(
+                doc_id=doc_id,
+                filename=filename,
+                executive_summary=summary,
+                key_entities=entities,
+                key_findings=findings,
+                topics=topics,
+            )
+            app_logger.info(f"[MEMORY] Saved knowledge dossier for doc_id={doc_id} ({filename})")
+
+            # Asynchronously update global knowledge summary
+            await self.synthesize_global_knowledge()
+
+            return {
+                "doc_id": doc_id,
+                "filename": filename,
+                "executive_summary": summary,
+                "key_entities": entities,
+                "key_findings": findings,
+                "topics": topics,
+            }
+        except Exception as e:
+            app_logger.error(f"[MEMORY] Failed to generate document dossier for {doc_id}: {e}", exc_info=True)
+            return None
+
+    # ── 3. Global Knowledge Summary Across All Data ────────────────────────
+    async def synthesize_global_knowledge(self) -> Dict[str, Any]:
+        """
+        Synthesizes a collective master knowledge summary of all indexed documents in the workspace.
+        """
+        dossiers = storage.list_knowledge_dossiers()
+        total_docs = len(dossiers)
+
+        if not dossiers:
+            # Check if any documents exist in doc_meta without dossiers yet
+            all_docs = storage.list_documents()
+            if not all_docs:
+                default_summary = "No documents currently indexed in the knowledge base."
+                storage.save_global_knowledge_summary(default_summary, 0, [])
+                return {"collective_summary": default_summary, "total_documents": 0, "key_insights": []}
+
+        # Build combined summary representation of all dossiers
+        dossier_summaries = []
+        for d in dossiers:
+            fname = d.get("filename", "Doc")
+            summ = d.get("executive_summary", "")[:600]
+            findings = "; ".join(d.get("key_findings", [])[:3])
+            topics = ", ".join(d.get("topics", [])[:4])
+            dossier_summaries.append(
+                f"- **{fname}**\n  Summary: {summ}\n  Topics: {topics}\n  Findings: {findings}"
+            )
+
+        combined_input = "\n\n".join(dossier_summaries)
+
+        prompt = f"""You are the Chief Knowledge Architect for Atlas Document Intelligence.
+Synthesize the collective knowledge across all {total_docs} documents ingested into the platform.
+
+INDEXED DOCUMENTS DOSSIERS:
+{combined_input}
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+OUTPUT REQUIREMENTS
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Return ONLY valid JSON with this exact schema:
+{{
+  "collective_summary": "A cohesive 2-4 paragraph overview of the global knowledge contained in all documents, highlighting overarching themes, areas of synergy, and domain expertise.",
+  "key_insights": [
+    "Cross-cutting insight 1",
+    "Cross-cutting insight 2",
+    "Cross-cutting insight 3"
+  ]
+}}
+"""
+        raw_result = ""
+        if not self.llm.hf_exhausted:
+            try:
+                resp = await self.llm.client.chat_completion(
+                    messages=[{"role": "user", "content": prompt}],
+                    model=self.llm.chat_model,
+                    temperature=0.3,
+                    max_tokens=1000,
+                )
+                raw_result = resp.choices[0].message.content or ""
+            except Exception as e:
+                self.llm._is_hf_exhausted(e)
+                app_logger.warning(f"HF global knowledge synthesis failed ({e}). Checking Groq fallback...")
+
+        if (not raw_result or self.llm.hf_exhausted) and self.llm.groq_client:
+            try:
+                groq_resp = await self.llm.groq_client.chat.completions.create(
+                    model=self.llm.groq_chat_model,
+                    messages=[{"role": "user", "content": prompt}],
+                    temperature=0.3,
+                    max_tokens=1000,
+                )
+                raw_result = groq_resp.choices[0].message.content or ""
+            except Exception as ge:
+                app_logger.error(f"Groq global knowledge synthesis failed: {ge}")
+
+        data = self.llm._extract_json(raw_result)
+        collective = data.get("collective_summary") or "Atlas Global Knowledge Base synthesis complete."
+        insights = data.get("key_insights") or []
+
+        storage.save_global_knowledge_summary(collective, total_docs, insights)
+        app_logger.info(f"[MEMORY] Successfully synthesized global knowledge across {total_docs} documents.")
+
+        return {
+            "collective_summary": collective,
+            "total_documents": total_docs,
+            "key_insights": insights,
+            "updated_at": time.time(),
+        }
+
+    def get_global_knowledge(self) -> Dict[str, Any]:
+        """Fetch current global knowledge summary and document dossiers."""
+        global_rec = storage.get_global_knowledge_summary() or {}
+        dossiers = storage.list_knowledge_dossiers()
+        return {
+            "collective_summary": global_rec.get(
+                "collective_summary", "No collective summary available yet. Ingest documents to build knowledge."
+            ),
+            "total_documents": global_rec.get("total_documents", len(dossiers)),
+            "key_insights": global_rec.get("key_insights", []),
+            "updated_at": global_rec.get("updated_at"),
+            "dossiers": dossiers,
+        }

test_memory_multidoc.py

@@ -0,0 +1,122 @@
+import os
+import sys
+
+# Add app to path
+app_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "app"))
+if app_dir not in sys.path:
+    sys.path.insert(0, app_dir)
+
+from fastapi.testclient import TestClient
+from app import app
+from core.storage import storage
+from core.persistent_db import persistent_db
+
+client = TestClient(app)
+
+def test_full_memory_and_multidoc_isolation():
+    print("\n--- 1. Testing Session Creation & Isolation ---")
+    res_a = client.post("/chat/new")
+    assert res_a.status_code == 200
+    chat_a = res_a.json()["chat_id"]
+
+    res_b = client.post("/chat/new")
+    assert res_b.status_code == 200
+    chat_b = res_b.json()["chat_b" if "chat_b" in res_b.json() else "chat_id"]
+
+    print(f"Chat A ID: {chat_a}")
+    print(f"Chat B ID: {chat_b}")
+    assert chat_a != chat_b
+
+    # Create dummy mock docs in storage
+    doc1_id = "mock-doc-1"
+    doc2_id = "mock-doc-2"
+    mock_tree_1 = [
+        {"title": "Section 1: Architecture", "page": 1, "text": "Atlas uses tree-based retrieval."},
+        {"title": "Section 2: Memory", "page": 2, "text": "Session memory isolates short and long context."},
+    ]
+    mock_tree_2 = [
+        {"title": "Overview: Financials", "page": 1, "text": "Revenue grew by 24% in Q3."},
+    ]
+
+    storage.set_document_ready(doc1_id, tree=mock_tree_1, filename="Architecture_Guide.pdf", page_count=2, section_count=2)
+    storage.set_document_ready(doc2_id, tree=mock_tree_2, filename="Financial_Report_Q3.pdf", page_count=1, section_count=1)
+
+    print("\n--- 2. Testing Multi-Document Attachment to Chat A ---")
+    attach_res = client.post(f"/chat/{chat_a}/attach-multiple", json={"doc_ids": [doc1_id, doc2_id]})
+    assert attach_res.status_code == 200
+    assert len(attach_res.json()["doc_ids"]) == 2
+
+    docs_a_res = client.get(f"/chat/{chat_a}/documents")
+    assert docs_a_res.status_code == 200
+    docs_a = docs_a_res.json()["documents"]
+    print(f"Chat A attached documents count: {len(docs_a)}")
+    assert len(docs_a) == 2
+    assert {d["doc_id"] for d in docs_a} == {doc1_id, doc2_id}
+
+    # Verify Chat B has zero attached documents (Strict Session Isolation)
+    docs_b_res = client.get(f"/chat/{chat_b}/documents")
+    assert docs_b_res.status_code == 200
+    docs_b = docs_b_res.json()["documents"]
+    print(f"Chat B attached documents count: {len(docs_b)}")
+    assert len(docs_b) == 0, "Violation of session isolation: Chat B should have 0 docs"
+
+    print("\n--- 3. Testing Message Isolation Between Sessions ---")
+    storage.add_message(chat_a, "user", "What is Atlas?")
+    storage.add_message(chat_a, "assistant", "Atlas is a document intelligence platform.")
+
+    storage.add_message(chat_b, "user", "Hello from user B")
+    storage.add_message(chat_b, "assistant", "Greetings, this is session B.")
+
+    hist_a = client.get(f"/chat/{chat_a}/history").json()["history"]
+    hist_b = client.get(f"/chat/{chat_b}/history").json()["history"]
+    assert len(hist_a) == 2
+    assert hist_a[0]["content"] == "What is Atlas?"
+    assert len(hist_b) == 2
+    assert hist_b[0]["content"] == "Hello from user B"
+    print("Session history isolation verified!")
+
+    print("\n--- 4. Testing Session Memory Diagnostic Endpoint ---")
+    mem_res = client.get(f"/memory/chat/{chat_a}")
+    assert mem_res.status_code == 200
+    mem_data = mem_res.json()
+    assert mem_data["chat_id"] == chat_a
+    assert mem_data["recent_turns_count"] == 2
+    print(f"Memory diagnostic for Chat A: {mem_data['recent_turns_count']} turns recorded.")
+
+    print("\n--- 5. Testing Detach Single Document ---")
+    detach_res = client.delete(f"/chat/{chat_a}/document/{doc1_id}")
+    assert detach_res.status_code == 200
+    remaining_docs = client.get(f"/chat/{chat_a}/documents").json()["documents"]
+    assert len(remaining_docs) == 1
+    assert remaining_docs[0]["doc_id"] == doc2_id
+    print(f"Remaining document in Chat A: {remaining_docs[0]['filename']}")
+
+    print("\n--- 6. Testing Global Knowledge & Dossier Storage ---")
+    # Save a test dossier
+    persistent_db.save_knowledge_dossier(
+        doc_id=doc1_id,
+        filename="Architecture_Guide.pdf",
+        executive_summary="Explains Atlas tree-based vectorless RAG architecture.",
+        key_entities=["Atlas", "PageIndex", "Groq", "FastAPI"],
+        key_findings=["Tree search outperforms embeddings on dense tabular texts."],
+        topics=["Architecture", "Document Intelligence", "FastAPI"]
+    )
+
+    persistent_db.save_global_knowledge_summary(
+        collective_summary="Atlas workspace contains technical documentation on tree retrieval and quarterly financial reports.",
+        total_documents=2,
+        key_insights=["Tree retrieval avoids embedding drift", "Multi-doc synthesis enables cross-sectional queries"]
+    )
+
+    g_res = client.get("/memory/knowledge")
+    assert g_res.status_code == 200
+    g_data = g_res.json()
+    print(f"Global Knowledge Summary: {g_data['collective_summary']}")
+    print(f"Dossiers count: {len(g_data['dossiers'])}")
+    assert len(g_data["dossiers"]) >= 1
+    assert "Atlas" in g_data["collective_summary"]
+
+    print("\n ALL BACKEND MULTI-DOC & MEMORY TESTS PASSED PERFECTLY!")
+
+if __name__ == "__main__":
+    test_full_memory_and_multidoc_isolation()