Add admin panel backend: upload/edit/delete endpoints + .docx support

- New admin API endpoints: POST /admin/upload, GET/DELETE /admin/documents,
GET /admin/categories, GET/PUT /admin/documents/content (edit + re-ingest)
- ingestion_service: add _chunk_docx(), category metadata on all chunks,
ingest_single_file(), delete_document(), UPLOAD_ALLOWED_EXTENSIONS
- vector_store: add list_sources() with path-derived category fallback
for documents ingested before the category field was added
- schemas: DocumentInfo, DocumentListResponse, UploadResponse,
CategoryListResponse, DocumentContentResponse, UpdateContentRequest
- requirements: add python-docx==1.1.2

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/.env.example

@@ -9,3 +9,45 @@ CHUNK_SIZE=700
 CHUNK_OVERLAP=100
 MAX_HISTORY_TURNS=10
 KNOWLEDGE_BASE_PATH=../knowledge
+
+# ── Admin panel ───────────────────────────────────────────────────────────────
+# Required to call all /api/admin/* endpoints (upload, list, delete).
+# Empty = all admin endpoints return 403.  Generate: openssl rand -hex 32
+ADMIN_TOKEN=
+# Add the admin panel's Vercel URL here so the browser can reach the backend.
+# Example: CORS_ORIGINS=http://localhost:3000,https://negoptim-admin.vercel.app
+CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000
+
+# ── Abuse protection / cost control ──────────────────────────────────────────
+# Shared secret gating POST /chat (must match the frontend proxy's value).
+# Empty = disabled (local dev). Generate with: openssl rand -hex 32
+INTERNAL_AUTH_TOKEN=
+RATE_LIMIT_CHAT_PER_MINUTE=20
+CHAT_PER_IP_PER_DAY=60
+CHAT_GLOBAL_PER_DAY=800
+MAX_INPUT_CHARS=1500
+MAX_OUTPUT_TOKENS=600
+
+# ── ROI Snapshot analysis feature ────────────────────────────────────────────
+ANALYSIS_ENABLED=true
+ANALYSIS_PER_IP_PER_DAY=2
+ANALYSIS_GLOBAL_PER_DAY=100
+ANALYSIS_MAX_OUTPUT_TOKENS=450
+BENCHMARK_DATA_TTL_HOURS=24
+
+# ── Email ─────────────────────────────────────────────────────────────────────
+# Delivery priority: Resend HTTP API → SMTP → simulation (all empty = logs only).
+# Resend is the only path that works on the HF free tier (SMTP ports blocked).
+RESEND_API_KEY=
+# Sales inbox: where demo requests are addressed. REQUIRED for real delivery.
+# With the Resend sandbox sender (onboarding@resend.dev), only the Resend
+# account owner's address is deliverable until a domain is verified.
+SALES_EMAIL=you@example.com
+# General email mode may only target these domains (comma-separated) —
+# open-relay protection for the public demo.
+EMAIL_RECIPIENT_ALLOWLIST=usersloveit.com
+# Force-route ALL outgoing mail to a test inbox. Empty = real recipients.
+EMAIL_OVERRIDE_TO=
+# SMTP fallback (local dev only)
+SMTP_USER=
+SMTP_PASSWORD=

backend/app/api/ingestion.py

@@ -1,29 +1,210 @@
+import hmac
+import re
 import logging
+from pathlib import Path
 
-from fastapi import APIRouter, Header, HTTPException
+from fastapi import APIRouter, Header, HTTPException, UploadFile, File, Form
 
 from app.config import settings
-from app.models.schemas import IngestRequest, IngestResponse
-from app.services.ingestion_service import ingest_folder
+from app.models.schemas import (
+    IngestRequest,
+    IngestResponse,
+    DocumentInfo,
+    DocumentListResponse,
+    UploadResponse,
+    CategoryListResponse,
+    DocumentContentResponse,
+    UpdateContentRequest,
+)
+from app.services.ingestion_service import (
+    ingest_folder,
+    ingest_single_file,
+    delete_document,
+    UPLOAD_ALLOWED_EXTENSIONS,
+)
+from app.services.vector_store import get_vector_store
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.post("/ingest", response_model=IngestResponse)
-async def ingest(request: IngestRequest, x_admin_token: str | None = Header(default=None)):
-    # Ingestion is CPU-heavy and mutates the index — never expose it publicly.
-    # In production the index is baked into the Docker image at build time, so
-    # this endpoint stays disabled unless ADMIN_TOKEN is explicitly configured.
-    if not settings.admin_token or x_admin_token != settings.admin_token:
+def _require_admin(token: str | None) -> None:
+    """Raise 403 if the token is missing or doesn't match ADMIN_TOKEN."""
+    if not settings.admin_token or not hmac.compare_digest(
+        (token or "").encode(), settings.admin_token.encode()
+    ):
         raise HTTPException(
             status_code=403,
-            detail="Ingestion over HTTP is disabled. Set ADMIN_TOKEN and pass the X-Admin-Token header, or run scripts/ingest.py locally.",
+            detail="Invalid or missing admin token. Set ADMIN_TOKEN and pass X-Admin-Token header.",
         )
 
+
+# ── Existing bulk-ingest endpoint ─────────────────────────────────────────────
+
+
+@router.post("/ingest", response_model=IngestResponse)
+async def ingest(
+    request: IngestRequest,
+    x_admin_token: str | None = Header(default=None),
+):
+    _require_admin(x_admin_token)
     try:
         count, message = ingest_folder(request.folder)
         return IngestResponse(ingested=count, message=message)
     except Exception:
         logger.exception("Ingestion failed")
         raise HTTPException(status_code=500, detail="Ingestion failed — see server logs.")
+
+
+# ── Admin: upload a single document ──────────────────────────────────────────
+
+
+@router.post("/admin/upload", response_model=UploadResponse)
+async def upload_document(
+    file: UploadFile = File(...),
+    category: str = Form(...),
+    x_admin_token: str | None = Header(default=None),
+):
+    _require_admin(x_admin_token)
+
+    ext = Path(file.filename or "").suffix.lower()
+    if ext not in UPLOAD_ALLOWED_EXTENSIONS:
+        allowed = ", ".join(sorted(UPLOAD_ALLOWED_EXTENSIONS))
+        raise HTTPException(
+            status_code=400,
+            detail=f"Unsupported file type '{ext}'. Allowed: {allowed}",
+        )
+
+    # Sanitize category: lowercase alphanumeric, hyphens, underscores only
+    safe_category = re.sub(r"[^\w\-]", "_", category.strip()).strip("_").lower()
+    if not safe_category:
+        raise HTTPException(status_code=400, detail="Category name is required")
+
+    # Sanitize filename: keep word chars, hyphens, dots, spaces → underscores
+    original_name = Path(file.filename or "upload").name
+    safe_name = re.sub(r"[^\w\-. ]", "", original_name).strip().replace(" ", "_")
+    if not safe_name or safe_name.startswith("."):
+        raise HTTPException(status_code=400, detail="Invalid filename")
+
+    base = Path(settings.knowledge_base_path).resolve()
+    category_dir = base / safe_category
+    category_dir.mkdir(parents=True, exist_ok=True)
+
+    file_path = category_dir / safe_name
+    content = await file.read()
+    file_path.write_bytes(content)
+
+    try:
+        count, message = ingest_single_file(file_path, base)
+    except Exception:
+        file_path.unlink(missing_ok=True)
+        logger.exception("Ingestion failed for %s", file_path)
+        raise HTTPException(status_code=500, detail="Ingestion failed — see server logs.")
+
+    return UploadResponse(
+        filename=safe_name,
+        category=safe_category,
+        chunks_ingested=count,
+        message=message,
+    )
+
+
+# ── Admin: list indexed documents ─────────────────────────────────────────────
+
+
+@router.get("/admin/documents", response_model=DocumentListResponse)
+async def list_documents(x_admin_token: str | None = Header(default=None)):
+    _require_admin(x_admin_token)
+    docs = get_vector_store().list_sources()
+    return DocumentListResponse(
+        documents=[DocumentInfo(**d) for d in docs],
+        total=len(docs),
+    )
+
+
+# ── Admin: delete a document ──────────────────────────────────────────────────
+
+
+@router.delete("/admin/documents")
+async def remove_document(
+    source: str,
+    x_admin_token: str | None = Header(default=None),
+):
+    _require_admin(x_admin_token)
+    ok, message = delete_document(source)
+    if not ok:
+        raise HTTPException(status_code=400, detail=message)
+    return {"message": message}
+
+
+# ── Admin: get editable file content ─────────────────────────────────────────
+
+
+@router.get("/admin/documents/content", response_model=DocumentContentResponse)
+async def get_document_content(
+    source: str,
+    x_admin_token: str | None = Header(default=None),
+):
+    _require_admin(x_admin_token)
+    base = Path(settings.knowledge_base_path).resolve()
+    target = (base / source).resolve()
+
+    if not target.is_relative_to(base):
+        raise HTTPException(status_code=400, detail="Invalid source path")
+    if not target.exists():
+        raise HTTPException(status_code=404, detail="File not found")
+    if target.suffix.lower() not in {".md", ".txt"}:
+        raise HTTPException(status_code=400, detail="Editing is only supported for .md and .txt files")
+
+    try:
+        content = target.read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        content = target.read_text(encoding="latin-1")
+
+    return DocumentContentResponse(source=source, content=content)
+
+
+# ── Admin: save edited content and re-ingest ─────────────────────────────────
+
+
+@router.put("/admin/documents/content", response_model=UploadResponse)
+async def update_document_content(
+    request: UpdateContentRequest,
+    x_admin_token: str | None = Header(default=None),
+):
+    _require_admin(x_admin_token)
+    base = Path(settings.knowledge_base_path).resolve()
+    target = (base / request.source).resolve()
+
+    if not target.is_relative_to(base):
+        raise HTTPException(status_code=400, detail="Invalid source path")
+    if not target.exists():
+        raise HTTPException(status_code=404, detail="File not found")
+    if target.suffix.lower() not in {".md", ".txt"}:
+        raise HTTPException(status_code=400, detail="Editing is only supported for .md and .txt files")
+
+    target.write_text(request.content, encoding="utf-8")
+
+    try:
+        count, message = ingest_single_file(target, base)
+    except Exception:
+        logger.exception("Re-ingestion failed for %s", target)
+        raise HTTPException(status_code=500, detail="Re-ingestion failed — see server logs.")
+
+    return UploadResponse(
+        filename=target.name,
+        category=target.parent.name if target.parent != base else "",
+        chunks_ingested=count,
+        message=message,
+    )
+
+
+# ── Admin: list available categories ─────────────────────────────────────────
+
+
+@router.get("/admin/categories", response_model=CategoryListResponse)
+async def list_categories(x_admin_token: str | None = Header(default=None)):
+    _require_admin(x_admin_token)
+    base = Path(settings.knowledge_base_path).resolve()
+    categories = sorted(d.name for d in base.iterdir() if d.is_dir())
+    return CategoryListResponse(categories=categories)

backend/app/models/schemas.py

@@ -9,12 +9,16 @@ class ChatRequest(BaseModel):
     # Email-flow controls (set by the draft card's Send/Cancel buttons).
     email_action: Optional[str] = None          # "send" | "cancel"
     email_payload: Optional[dict] = None         # {"subject": ..., "body": ...} (edited draft)
+    # Analysis-flow control (set by the Negoptim AI toggle).
+    analysis_action: Optional[str] = None       # "start" | "cancel"
 
 
 class Source(BaseModel):
     document: str
     section: Optional[str] = None
     score: float
+    # Truncated retrieved chunk text, shown in the UI's source detail popup.
+    excerpt: Optional[str] = None
 
 
 class TokenUsage(BaseModel):
@@ -53,6 +57,42 @@ class IngestResponse(BaseModel):
     message: str
 
 
+# ── Admin schemas ─────────────────────────────────────────────────────────────
+
+
+class DocumentInfo(BaseModel):
+    source: str
+    category: str
+    title: str
+    chunk_count: int
+
+
+class DocumentListResponse(BaseModel):
+    documents: List[DocumentInfo]
+    total: int
+
+
+class UploadResponse(BaseModel):
+    filename: str
+    category: str
+    chunks_ingested: int
+    message: str
+
+
+class CategoryListResponse(BaseModel):
+    categories: List[str]
+
+
+class DocumentContentResponse(BaseModel):
+    source: str
+    content: str
+
+
+class UpdateContentRequest(BaseModel):
+    source: str
+    content: str
+
+
 class HealthResponse(BaseModel):
     status: str
     llm: bool

backend/app/services/ingestion_service.py

@@ -10,52 +10,78 @@ from app.services.vector_store import get_vector_store
 
 logger = logging.getLogger(__name__)
 
-SUPPORTED_EXTENSIONS = {".md", ".txt", ".pdf"}
+SUPPORTED_EXTENSIONS = {".md", ".txt", ".pdf", ".docx"}
+# Subset accepted through the admin upload endpoint (PDF excluded for now)
+UPLOAD_ALLOWED_EXTENSIONS = {".md", ".txt", ".docx"}
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+
+def _title_from_path(source_path: str) -> str:
+    return Path(source_path).stem.replace("-", " ").replace("_", " ").strip().title()
+
+
+def _extract_category(source_path: str) -> str:
+    """Top-level folder name becomes the document category."""
+    parts = Path(source_path).parts
+    return parts[0] if len(parts) > 1 else ""
 
 
 # ── Chunking ──────────────────────────────────────────────────────────────────
 
 
 def _chunk_markdown(text: str, source_path: str) -> List[Dict[str, Any]]:
-    """Split markdown by H2/H3 headings, further split oversized sections."""
-    chunks: List[Dict[str, Any]] = []
-    # Split on lines that start with ## or ###
-    sections = re.split(r"(?m)^(#{2,3} .+)$", text)
+    """Split markdown by heading structure (H1-H3), tracking a breadcrumb
+    trail so each chunk knows its place in the document hierarchy."""
+    parts = re.split(r"(?m)^(#{1,3} .+)$", text)
 
-    current_heading = ""
-    buffer = ""
+    title = ""
+    h2 = h3 = ""
+    chunks: List[Dict[str, Any]] = []
 
-    def flush(heading: str, content: str) -> None:
+    def flush(content: str) -> None:
         content = content.strip()
         if not content:
             return
-        sub_chunks = _split_by_size(content, heading)
-        chunks.extend(sub_chunks)
+        section = " › ".join(h for h in (h2, h3) if h)
+        for piece in _split_by_size(content):
+            chunks.append({"text": piece, "section": section})
 
-    for part in sections:
-        if re.match(r"^#{2,3} .+$", part.strip()):
-            flush(current_heading, buffer)
-            current_heading = part.strip().lstrip("#").strip()
+    buffer = ""
+    for part in parts:
+        m = re.match(r"^(#{1,3}) (.+)$", part.strip())
+        if m:
+            flush(buffer)
+            level, heading = len(m.group(1)), m.group(2).strip()
+            if level == 1:
+                title = title or heading
+                h2 = h3 = ""
+            elif level == 2:
+                h2, h3 = heading, ""
+            else:
+                h3 = heading
             buffer = part + "\n"
         else:
             buffer += part
+    flush(buffer)
 
-    flush(current_heading, buffer)
-
-    result = []
-    for i, chunk in enumerate(chunks):
-        result.append(
-            {
-                "text": chunk["text"],
-                "metadata": {
-                    "source": source_path,
-                    "section": chunk.get("heading", ""),
-                    "chunk_index": i,
-                    "type": "markdown",
-                },
-            }
-        )
-    return result
+    if not title:
+        title = _title_from_path(source_path)
+
+    return [
+        {
+            "text": c["text"],
+            "metadata": {
+                "source": source_path,
+                "title": title,
+                "section": c["section"],
+                "chunk_index": i,
+                "type": "markdown",
+            },
+        }
+        for i, c in enumerate(chunks)
+    ]
 
 
 def _chunk_text(text: str, source_path: str) -> List[Dict[str, Any]]:
@@ -68,6 +94,7 @@ def _chunk_text(text: str, source_path: str) -> List[Dict[str, Any]]:
             "text": chunk,
             "metadata": {
                 "source": source_path,
+                "title": _title_from_path(source_path),
                 "section": "",
                 "chunk_index": i,
                 "type": "text",
@@ -92,28 +119,135 @@ def _chunk_pdf(file_path: Path, source_path: str) -> List[Dict[str, Any]]:
     return _chunk_text(full_text, source_path)
 
 
-def _split_by_size(text: str, heading: str) -> List[Dict[str, Any]]:
+def _chunk_docx(file_path: Path, source_path: str) -> List[Dict[str, Any]]:
+    """Extract text from a Word document, treating Heading styles like markdown
+    headings so structure is preserved in the same way as .md files."""
+    try:
+        from docx import Document as DocxDocument
+    except ImportError:
+        logger.warning("python-docx not installed, skipping .docx: %s", file_path)
+        return []
+
+    doc = DocxDocument(str(file_path))
+
+    # Parse paragraphs into (level, text): level 0 = body, 1/2/3 = headings
+    entries: List[Tuple[int, str]] = []
+    for para in doc.paragraphs:
+        text = para.text.strip()
+        if not text:
+            continue
+        style = para.style.name if para.style else ""
+        if style.startswith("Heading 1"):
+            entries.append((1, text))
+        elif style.startswith("Heading 2"):
+            entries.append((2, text))
+        elif style.startswith("Heading 3"):
+            entries.append((3, text))
+        else:
+            entries.append((0, text))
+
+    title = ""
+    h2 = h3 = ""
+    raw_chunks: List[Dict[str, Any]] = []
+    buffer: List[str] = []
+
+    def flush_buffer() -> None:
+        combined = "\n\n".join(buffer).strip()
+        buffer.clear()
+        if not combined:
+            return
+        section = " › ".join(h for h in (h2, h3) if h)
+        for piece in _split_by_size(combined):
+            raw_chunks.append({"text": piece, "section": section})
+
+    for level, text in entries:
+        if level == 0:
+            buffer.append(text)
+        elif level == 1:
+            flush_buffer()
+            if not title:
+                title = text
+            h2 = h3 = ""
+        elif level == 2:
+            flush_buffer()
+            h2, h3 = text, ""
+        else:
+            flush_buffer()
+            h3 = text
+
+    flush_buffer()
+
+    if not title:
+        title = _title_from_path(source_path)
+
+    return [
+        {
+            "text": c["text"],
+            "metadata": {
+                "source": source_path,
+                "title": title,
+                "section": c["section"],
+                "chunk_index": i,
+                "type": "docx",
+            },
+        }
+        for i, c in enumerate(raw_chunks)
+    ]
+
+
+def _split_by_size(text: str) -> List[str]:
     """Further split a section that exceeds chunk_size."""
     if len(text) <= settings.chunk_size:
-        return [{"text": text, "heading": heading}]
-
+        return [text]
     paragraphs = [p.strip() for p in re.split(r"\n{2,}", text) if p.strip()]
-    combined = _combine_paragraphs(paragraphs)
-    return [{"text": c, "heading": heading} for c in combined]
+    return _combine_paragraphs(paragraphs)
+
+
+def _split_oversized(paragraph: str) -> List[str]:
+    """Split a single paragraph that exceeds chunk_size at sentence boundaries."""
+    if len(paragraph) <= settings.chunk_size:
+        return [paragraph]
+
+    sentences = re.split(r"(?<=[.!?])\s+", paragraph)
+    pieces: List[str] = []
+    current = ""
+    for sentence in sentences:
+        candidate = f"{current} {sentence}".strip() if current else sentence
+        if len(candidate) > settings.chunk_size and current:
+            pieces.append(current)
+            current = sentence
+        else:
+            current = candidate
+    if current:
+        pieces.append(current)
+    return pieces
+
+
+def _overlap_tail(chunk: str) -> str:
+    """Last chunk_overlap chars of a chunk, trimmed to a word boundary."""
+    if not settings.chunk_overlap:
+        return ""
+    tail = chunk[-settings.chunk_overlap:]
+    if " " in tail:
+        tail = tail.split(" ", 1)[1]
+    return tail.strip()
 
 
 def _combine_paragraphs(paragraphs: List[str]) -> List[str]:
     """Combine short paragraphs into chunks of ~chunk_size with overlap."""
+    units: List[str] = []
+    for para in paragraphs:
+        units.extend(_split_oversized(para))
+
     chunks: List[str] = []
     current = ""
-    overlap_buffer = ""
 
-    for para in paragraphs:
+    for para in units:
         candidate = (current + "\n\n" + para).strip() if current else para
         if len(candidate) > settings.chunk_size and current:
             chunks.append(current.strip())
-            overlap_buffer = current[-settings.chunk_overlap:] if settings.chunk_overlap else ""
-            current = (overlap_buffer + "\n\n" + para).strip()
+            tail = _overlap_tail(current)
+            current = (tail + "\n\n" + para).strip() if tail else para
         else:
             current = candidate
 
@@ -123,7 +257,16 @@ def _combine_paragraphs(paragraphs: List[str]) -> List[str]:
     return chunks if chunks else ["\n\n".join(paragraphs)]
 
 
-# ── Ingestion ─────────────────────────────────────────────────────────────────
+# ── Ingestion helpers ─────────────────────────────────────────────────────────
+
+
+def _embedding_text(chunk: Dict[str, Any]) -> str:
+    """Text actually embedded: '<title> — <section>' header + chunk body."""
+    meta = chunk["metadata"]
+    header = meta.get("title", "")
+    if meta.get("section"):
+        header = f"{header} — {meta['section']}" if header else meta["section"]
+    return f"{header}\n{chunk['text']}" if header else chunk["text"]
 
 
 def _make_chunk_id(source_path: str, index: int, text: str) -> str:
@@ -135,26 +278,94 @@ def _make_chunk_id(source_path: str, index: int, text: str) -> str:
 def _process_file(file_path: Path, base_path: Path) -> List[Dict[str, Any]]:
     relative = str(file_path.relative_to(base_path))
     ext = file_path.suffix.lower()
+    category = _extract_category(relative)
 
     if ext == ".pdf":
-        return _chunk_pdf(file_path, relative)
+        chunks = _chunk_pdf(file_path, relative)
+    elif ext == ".docx":
+        chunks = _chunk_docx(file_path, relative)
+    else:
+        try:
+            text = file_path.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            text = file_path.read_text(encoding="latin-1")
+        if ext == ".md":
+            chunks = _chunk_markdown(text, relative)
+        else:
+            chunks = _chunk_text(text, relative)
 
-    try:
-        text = file_path.read_text(encoding="utf-8")
-    except UnicodeDecodeError:
-        text = file_path.read_text(encoding="latin-1")
+    for c in chunks:
+        c["metadata"]["category"] = category
+
+    return chunks
 
-    if ext == ".md":
-        return _chunk_markdown(text, relative)
-    return _chunk_text(text, relative)
+
+def _ingest_chunks(chunks: List[Dict[str, Any]], vector_store, embedding_svc) -> int:
+    """Embed and upsert chunks in batches. Returns total upserted."""
+    BATCH = 64
+    total = 0
+    for i in range(0, len(chunks), BATCH):
+        batch = chunks[i: i + BATCH]
+        embeddings = embedding_svc.embed_passages([_embedding_text(c) for c in batch])
+        ids = [
+            _make_chunk_id(c["metadata"]["source"], c["metadata"]["chunk_index"], c["text"])
+            for c in batch
+        ]
+        vector_store.upsert(
+            ids=ids,
+            embeddings=embeddings,
+            documents=[c["text"] for c in batch],
+            metadatas=[c["metadata"] for c in batch],
+        )
+        total += len(batch)
+        logger.info("Ingested %d / %d chunks", total, len(chunks))
+    return total
+
+
+# ── Public API ────────────────────────────────────────────────────────────────
+
+
+def ingest_single_file(file_path: Path, base_path: Path) -> Tuple[int, str]:
+    """Ingest (or re-ingest) a single file. Deletes stale vectors first."""
+    if file_path.suffix.lower() not in SUPPORTED_EXTENSIONS:
+        return 0, f"Unsupported file type: {file_path.suffix}"
+
+    embedding_svc = get_embedding_service()
+    vector_store = get_vector_store()
+
+    chunks = _process_file(file_path, base_path)
+    if not chunks:
+        return 0, f"No content extracted from {file_path.name}"
+
+    relative = str(file_path.relative_to(base_path))
+    vector_store.delete_by_sources([relative])
+    total = _ingest_chunks(chunks, vector_store, embedding_svc)
+
+    return total, f"Ingested {total} chunks from {file_path.name}"
+
+
+def delete_document(source: str) -> Tuple[bool, str]:
+    """Remove a document's vectors and its file from the knowledge base."""
+    base = Path(settings.knowledge_base_path).resolve()
+    target = (base / source).resolve()
+
+    if not target.is_relative_to(base):
+        return False, "Invalid source path"
+
+    get_vector_store().delete_by_sources([source])
+
+    if target.exists() and target.is_file():
+        target.unlink()
+
+    return True, f"Deleted {source}"
 
 
 def ingest_folder(folder: str | None = None) -> Tuple[int, str]:
     base = Path(settings.knowledge_base_path).resolve()
 
     if folder:
-        target = base / folder
-        if not target.exists():
+        target = (base / folder).resolve()
+        if not target.is_relative_to(base) or not target.exists():
             return 0, f"Folder not found: {folder}"
         targets = [target]
     else:
@@ -169,22 +380,13 @@ def ingest_folder(folder: str | None = None) -> Tuple[int, str]:
         for file_path in target.rglob("*"):
             if file_path.suffix.lower() in SUPPORTED_EXTENSIONS and file_path.is_file():
                 logger.info("Processing: %s", file_path)
-                chunks = _process_file(file_path, base)
-                all_chunks.extend(chunks)
+                all_chunks.extend(_process_file(file_path, base))
 
     if not all_chunks:
         return 0, "No supported files found"
 
-    BATCH = 64
-    total = 0
-    for i in range(0, len(all_chunks), BATCH):
-        batch = all_chunks[i : i + BATCH]
-        texts = [c["text"] for c in batch]
-        embeddings = embedding_svc.embed_passages(texts)
-        ids = [_make_chunk_id(c["metadata"]["source"], c["metadata"]["chunk_index"], c["text"]) for c in batch]
-        metadatas = [c["metadata"] for c in batch]
-        vector_store.upsert(ids=ids, embeddings=embeddings, documents=texts, metadatas=metadatas)
-        total += len(batch)
-        logger.info("Ingested %d / %d chunks", total, len(all_chunks))
+    processed_sources = sorted({c["metadata"]["source"] for c in all_chunks})
+    vector_store.delete_by_sources(processed_sources)
 
+    total = _ingest_chunks(all_chunks, vector_store, embedding_svc)
     return total, f"Successfully ingested {total} chunks from {len(all_chunks)} total"

backend/app/services/vector_store.py

@@ -34,6 +34,14 @@ class VectorStore:
             metadatas=metadatas,
         )
 
+    def delete_by_sources(self, sources: List[str]) -> None:
+        """Remove all chunks belonging to the given source files. Called before
+        re-ingesting a file so edited documents don't leave stale vectors
+        behind (chunk IDs are content-hashed, so edits change the IDs)."""
+        if not sources:
+            return
+        self._collection.delete(where={"source": {"$in": sources}})
+
     def query(self, embedding: List[float], top_k: int) -> Dict[str, Any]:
         return self._collection.query(
             query_embeddings=[embedding],
@@ -52,6 +60,36 @@ class VectorStore:
         )
         logger.info("Collection reset")
 
+    def list_sources(self) -> List[Dict[str, Any]]:
+        """Return per-source stats aggregated from all chunk metadata.
+        Category is read from stored metadata when available, otherwise
+        derived from the top-level folder in the source path so that
+        documents ingested before the category field was added still
+        display correctly."""
+        from pathlib import Path as _Path
+
+        result = self._collection.get(include=["metadatas"])
+        metadatas = result.get("metadatas") or []
+
+        sources: Dict[str, Dict[str, Any]] = {}
+        for meta in metadatas:
+            source = meta.get("source", "")
+            stored_cat = meta.get("category", "")
+            if not stored_cat:
+                parts = _Path(source).parts
+                stored_cat = parts[0] if len(parts) > 1 else ""
+
+            if source not in sources:
+                sources[source] = {
+                    "source": source,
+                    "category": stored_cat,
+                    "title": meta.get("title", ""),
+                    "chunk_count": 0,
+                }
+            sources[source]["chunk_count"] += 1
+
+        return sorted(sources.values(), key=lambda x: x["source"])
+
     def is_healthy(self) -> bool:
         try:
             self._collection.count()

backend/requirements.txt

@@ -7,4 +7,5 @@ sentence-transformers==3.3.1
 httpx==0.28.1
 python-multipart==0.0.19
 pypdf==5.1.0
+python-docx==1.1.2
 python-dotenv==1.0.1