Fixed Storage

src/backend/ingest.py

@@ -65,33 +65,63 @@ def extract_url(url: str):
 def upsert_extracted(username: str, notebook_id: str, source_title: str, source_id: str, extracted_items: list[dict]) -> int:
     col = get_collection(username, notebook_id)
     ids, docs, metas = [], [], []
+
     for item in extracted_items:
+        page = item.get("page", None)
+        slide = item.get("slide", None)
+
         for j, ch in enumerate(simple_chunk(item["text"])):
             ids.append(f"{source_id}::chunk{j}")
             docs.append(ch)
-            metas.append({
-                "source_title": source_title,
-                "source_id": source_id,
-                "page": item.get("page"),
-                "slide": item.get("slide"),
-            })
+
+            meta = {
+                "source_title": str(source_title),
+                "source_id": str(source_id),
+            }
+            # IMPORTANT: Chroma metadata cannot include None
+            if page is not None:
+                meta["page"] = int(page)
+            if slide is not None:
+                meta["slide"] = int(slide)
+
+            metas.append(meta)
+
     if not docs:
         return 0
+
     embs = EMBED_MODEL.encode(docs, normalize_embeddings=True).tolist()
     col.upsert(ids=ids, documents=docs, metadatas=metas, embeddings=embs)
     return len(docs)
 
-def ingest_files(username: str, notebook_id: str, files) -> int:
+
+def ingest_files(username: str, notebook_id: str, filepaths) -> int:
+    """
+    filepaths may be:
+    - list[str]
+    - list[Gradio file objects]
+    """
     ensure_tree(username, notebook_id)
     raw_dir = os.path.join(nb_root(username, notebook_id), "files_raw")
     ex_dir = os.path.join(nb_root(username, notebook_id), "files_extracted")
     added = 0
 
-    for f in (files or []):
-        fp = getattr(f, "name", None)
-        if not fp:
-            continue
+    # Normalize gradio file objects -> local paths
+    normalized_paths = []
+    if isinstance(filepaths, (list, tuple)):
+        for f in filepaths:
+            if f is None:
+                continue
+            # Gradio may pass objects with .name
+            if hasattr(f, "name") and isinstance(f.name, str):
+                normalized_paths.append(f.name)
+            elif isinstance(f, str):
+                normalized_paths.append(f)
+            elif isinstance(f, dict) and "name" in f:
+                normalized_paths.append(f["name"])
+    elif isinstance(filepaths, str):
+        normalized_paths = [filepaths]
 
+    for fp in normalized_paths:
         dest = os.path.join(raw_dir, os.path.basename(fp))
         pathlib.Path(dest).write_bytes(pathlib.Path(fp).read_bytes())
 
@@ -105,15 +135,17 @@ def ingest_files(username: str, notebook_id: str, files) -> int:
         else:
             continue
 
+        # save extracted
         ex_path = os.path.join(ex_dir, os.path.basename(dest) + ".txt")
-        with open(ex_path, "w", encoding="utf-8") as ftxt:
+        with open(ex_path, "w", encoding="utf-8") as f:
             for item in extracted:
-                loc = ""
-                if item.get("page"):
+                if item.get("page") is not None:
                     loc = f"page={item.get('page')}"
-                elif item.get("slide"):
+                elif item.get("slide") is not None:
                     loc = f"slide={item.get('slide')}"
-                ftxt.write(f"\n--- {loc} ---\n{item['text']}\n")
+                else:
+                    loc = ""
+                f.write(f"\n--- {loc} ---\n{item['text']}\n")
 
         added += upsert_extracted(
             username,
@@ -124,6 +156,8 @@ def ingest_files(username: str, notebook_id: str, files) -> int:
         )
 
     return added
+
+
 def ingest_url(username: str, notebook_id: str, url: str) -> int:
     ensure_tree(username, notebook_id)
     extracted = extract_url(url)

src/backend/rag.py

@@ -11,10 +11,10 @@ def retrieve(username: str, notebook_id: str, query: str, k=6):
     qemb = EMBED_MODEL.encode([query], normalize_embeddings=True).tolist()
 
     res = col.query(
-        query_embeddings=qemb,
-        n_results=k,
-        include=["documents", "metadatas", "distances"],
-    )
+    query_embeddings=qemb,
+    n_results=k,
+    include=["documents", "metadatas", "distances"]
+)
 
     ids = res.get("ids", [[]])[0]
     docs = res.get("documents", [[]])[0]

src/storage/chroma_store.py

@@ -1,27 +1,19 @@
 import os
 import chromadb
-from src.storage.paths import nb_root
-
-
-def chroma_client(username, notebook_id):
 
-    persist_dir = os.path.join(
-        nb_root(username, notebook_id),
-        "chroma"
-    )
+from src.storage.paths import nb_root
 
-    return chromadb.Client(
-        chromadb.config.Settings(
-            persist_directory=persist_dir,
-            anonymized_telemetry=False
-        )
-    )
+# Cache clients by persist_dir to avoid "ephemeral with different settings"
+_CLIENTS: dict[str, chromadb.PersistentClient] = {}
 
+def chroma_client(username: str, notebook_id: str) -> chromadb.PersistentClient:
+    persist_dir = os.path.join(nb_root(username, notebook_id), "chroma")
+    os.makedirs(persist_dir, exist_ok=True)
 
-def get_collection(username, notebook_id):
+    if persist_dir not in _CLIENTS:
+        _CLIENTS[persist_dir] = chromadb.PersistentClient(path=persist_dir)
+    return _CLIENTS[persist_dir]
 
+def get_collection(username: str, notebook_id: str, name: str = "chunks"):
     client = chroma_client(username, notebook_id)
-
-    return client.get_or_create_collection(
-        name="notebook"
-    )
+    return client.get_or_create_collection(name=name)