Fix storage paths/index_path + chroma persistence + ingestion + telemetry

app.py

@@ -1,4 +1,8 @@
 import os
+
+# Disable Chroma telemetry noise
+os.environ["ANONYMIZED_TELEMETRY"] = "FALSE"
+
 from src.frontend.ui import build_app
 
 demo = build_app()

src/backend/ingest.py

@@ -81,13 +81,17 @@ def upsert_extracted(username: str, notebook_id: str, source_title: str, source_
     col.upsert(ids=ids, documents=docs, metadatas=metas, embeddings=embs)
     return len(docs)
 
-def ingest_files(username: str, notebook_id: str, filepaths: list[str]) -> int:
+def ingest_files(username: str, notebook_id: str, files) -> int:
     ensure_tree(username, notebook_id)
     raw_dir = os.path.join(nb_root(username, notebook_id), "files_raw")
     ex_dir = os.path.join(nb_root(username, notebook_id), "files_extracted")
     added = 0
 
-    for fp in filepaths:
+    for f in (files or []):
+        fp = getattr(f, "name", None)
+        if not fp:
+            continue
+
         dest = os.path.join(raw_dir, os.path.basename(fp))
         pathlib.Path(dest).write_bytes(pathlib.Path(fp).read_bytes())
 
@@ -101,17 +105,25 @@ def ingest_files(username: str, notebook_id: str, filepaths: list[str]) -> int:
         else:
             continue
 
-        # save extracted
         ex_path = os.path.join(ex_dir, os.path.basename(dest) + ".txt")
-        with open(ex_path, "w", encoding="utf-8") as f:
+        with open(ex_path, "w", encoding="utf-8") as ftxt:
             for item in extracted:
-                loc = f"page={item.get('page')}" if item.get("page") else f"slide={item.get('slide')}" if item.get("slide") else ""
-                f.write(f"\n--- {loc} ---\n{item['text']}\n")
+                loc = ""
+                if item.get("page"):
+                    loc = f"page={item.get('page')}"
+                elif item.get("slide"):
+                    loc = f"slide={item.get('slide')}"
+                ftxt.write(f"\n--- {loc} ---\n{item['text']}\n")
 
-        added += upsert_extracted(username, notebook_id, os.path.basename(dest), f"file:{os.path.basename(dest)}", extracted)
+        added += upsert_extracted(
+            username,
+            notebook_id,
+            os.path.basename(dest),
+            f"file:{os.path.basename(dest)}",
+            extracted,
+        )
 
     return added
-
 def ingest_url(username: str, notebook_id: str, url: str) -> int:
     ensure_tree(username, notebook_id)
     extracted = extract_url(url)

src/backend/notebooks.py

@@ -1,49 +1,44 @@
+import os
+import shutil
 import uuid
-import gradio as gr
-from src.storage.index_store import load_index, save_index, list_notebooks
-from src.storage.paths import ensure_tree
-from src.utils.text import safe_name
 from datetime import datetime
 
-def now_iso():
+from src.storage.index_store import load_index, save_index
+from src.storage.paths import nb_root, ensure_tree
+
+
+def _now():
     return datetime.utcnow().isoformat() + "Z"
 
+
 def create_notebook(username: str, name: str) -> str:
-    name = safe_name(name)
-    idx = load_index(username)
     nb_id = str(uuid.uuid4())
-    idx["notebooks"].append({
-        "id": nb_id,
-        "name": name,
-        "created_at": now_iso(),
-        "updated_at": now_iso(),
-    })
+
+    idx = load_index(username)
+    idx.append({"id": nb_id, "name": name or "Untitled", "created_at": _now(), "updated_at": _now()})
     save_index(username, idx)
+
     ensure_tree(username, nb_id)
     return nb_id
 
+
 def rename_notebook(username: str, notebook_id: str, new_name: str):
-    new_name = safe_name(new_name)
-    if not new_name:
-        raise gr.Error("Notebook name cannot be empty.")
     idx = load_index(username)
-    found = False
-    for nb in idx.get("notebooks", []):
+    for nb in idx:
         if nb["id"] == notebook_id:
             nb["name"] = new_name
-            nb["updated_at"] = now_iso()
-            found = True
+            nb["updated_at"] = _now()
             break
-    if not found:
-        raise gr.Error("Notebook not found.")
     save_index(username, idx)
 
+
 def delete_notebook(username: str, notebook_id: str):
-    import shutil, os
-    from src.storage.paths import nb_root
+    # remove folder
+    p = nb_root(username, notebook_id)
+    if os.path.exists(p):
+        shutil.rmtree(p, ignore_errors=True)
+
+    # remove from index
     idx = load_index(username)
-    idx["notebooks"] = [n for n in idx.get("notebooks", []) if n["id"] != notebook_id]
-    save_index(username, idx)
-    base = nb_root(username, notebook_id)
-    if os.path.exists(base):
-        shutil.rmtree(base, ignore_errors=True)
+    idx = [nb for nb in idx if nb.get("id") != notebook_id]
+    save_index(username, idx)

src/backend/rag.py

@@ -8,81 +8,61 @@ EMBED_MODEL = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 def retrieve(username: str, notebook_id: str, query: str, k=6):
     col = get_collection(username, notebook_id)
 
-    qemb = EMBED_MODEL.encode(
-        [query],
-        normalize_embeddings=True
-    ).tolist()
+    qemb = EMBED_MODEL.encode([query], normalize_embeddings=True).tolist()
 
-    # Compatible with HF Chroma version
     res = col.query(
         query_embeddings=qemb,
         n_results=k,
-        include=["documents", "metadatas", "distances"]
+        include=["documents", "metadatas", "distances"],
     )
 
-    # Safe extraction (prevents crashes)
     ids = res.get("ids", [[]])[0]
     docs = res.get("documents", [[]])[0]
     mets = res.get("metadatas", [[]])[0]
     dists = res.get("distances", [[]])[0]
 
     hits = []
-
     for i in range(len(docs)):
-        hits.append({
-            "id": ids[i] if i < len(ids) else f"chunk_{i}",
-            "doc": docs[i],
-            "meta": mets[i] if i < len(mets) else {},
-            "distance": dists[i] if i < len(dists) else None
-        })
-
+        hits.append(
+            {
+                "id": ids[i] if i < len(ids) else f"chunk_{i}",
+                "doc": docs[i],
+                "meta": mets[i] if i < len(mets) else {},
+                "distance": dists[i] if i < len(dists) else None,
+            }
+        )
     return hits
 
 
 def format_sources(hits):
     lines = []
-
     for i, h in enumerate(hits, start=1):
-        m = h["meta"] or {}
-
+        m = h.get("meta") or {}
+        title = m.get("source_title", "source")
         loc = ""
         if m.get("page"):
             loc = f"p.{m['page']}"
-
         if m.get("slide"):
             loc = f"slide {m['slide']}"
-
-        title = m.get("source_title", "source")
-
         lines.append(f"[S{i}] {title} {loc}".strip())
-
     return "\n".join(lines)
 
 
 def context_block(hits):
     blocks = []
-
     for i, h in enumerate(hits, start=1):
-        m = h["meta"] or {}
-
+        m = h.get("meta") or {}
+        title = m.get("source_title", "source")
         loc = ""
         if m.get("page"):
             loc = f"(page {m['page']})"
-
         if m.get("slide"):
             loc = f"(slide {m['slide']})"
-
-        title = m.get("source_title", "source")
-
-        blocks.append(
-            f"[S{i}] {title} {loc}\n{h['doc']}"
-        )
-
+        blocks.append(f"[S{i}] {title} {loc}\n{h.get('doc','')}")
     return "\n\n---\n\n".join(blocks)
 
 
 def rag_answer(query: str, hits):
-
     if not hits:
         return "Not found in the provided sources. (No indexed chunks yet.)"
 
@@ -90,31 +70,19 @@ def rag_answer(query: str, hits):
 You are a research assistant.
 
 Answer ONLY using the sources below.
-
 Every non-trivial claim must end with citations like [S1] or [S2].
-
-If not present in sources say:
-Not found in the provided sources.
+If not present in sources, say: Not found in the provided sources.
 
 Question:
 {query}
 
-
 Sources list:
 {format_sources(hits)}
 
-
 Source excerpts:
 {context_block(hits)}
 
-
 Answer with citations:
 """
-
-    ans = llm_generate(
-        prompt,
-        max_new_tokens=450,
-        temperature=0.2
-    )
-
+    ans = llm_generate(prompt, max_new_tokens=450, temperature=0.2)
     return f"{ans}\n\nSources:\n{format_sources(hits)}"

src/storage/index_store.py

@@ -1,31 +1,32 @@
-import os, json
-from datetime import datetime
-from .paths import user_root, index_path, ensure_tree
+import json
+import os
+from .paths import user_root, index_path
 
-def now_iso():
-    return datetime.utcnow().isoformat() + "Z"
 
-def load_index(username: str) -> dict:
+def load_index(username: str):
     os.makedirs(user_root(username), exist_ok=True)
-    p = index_path(username)
-    if not os.path.exists(p):
-        with open(p, "w", encoding="utf-8") as f:
-            json.dump({"notebooks": []}, f, indent=2)
-    with open(p, "r", encoding="utf-8") as f:
+    ip = index_path(username)
+    if not os.path.exists(ip):
+        with open(ip, "w", encoding="utf-8") as f:
+            f.write("[]")
+        return []
+    with open(ip, "r", encoding="utf-8") as f:
         return json.load(f)
 
-def save_index(username: str, idx: dict):
-    with open(index_path(username), "w", encoding="utf-8") as f:
-        json.dump(idx, f, indent=2)
 
-def list_notebooks(username: str):
-    idx = load_index(username)
-    return [(nb["name"], nb["id"]) for nb in idx.get("notebooks", [])]
+def save_index(username: str, items):
+    os.makedirs(user_root(username), exist_ok=True)
+    ip = index_path(username)
+    with open(ip, "w", encoding="utf-8") as f:
+        json.dump(items, f, ensure_ascii=False, indent=2)
 
-def touch_updated(username: str, notebook_id: str):
+
+def list_notebooks(username: str):
+    """
+    Returns list of tuples (label, id) for gr.Dropdown choices.
+    """
     idx = load_index(username)
-    for nb in idx.get("notebooks", []):
-        if nb["id"] == notebook_id:
-            nb["updated_at"] = now_iso()
-            break
-    save_index(username, idx)
+    out = []
+    for nb in idx:
+        out.append((nb.get("name", "Untitled"), nb.get("id")))
+    return out

src/storage/paths.py

@@ -1,18 +1,41 @@
 import os
 
+# HF Spaces uses /data; locally it can fall back to ./data
 DATA_ROOT = os.getenv("DATA_ROOT", "./data")
 
-def user_root(username: str):
-    return os.path.join(DATA_ROOT, username)
 
-def nb_root(username: str, notebook_id: str):
+def user_root(username: str) -> str:
+    return os.path.join(DATA_ROOT, "users", username, "notebooks")
+
+
+def index_path(username: str) -> str:
+    # /data/users/<username>/notebooks/index.json
+    return os.path.join(user_root(username), "index.json")
+
+
+def nb_root(username: str, notebook_id: str) -> str:
+    # /data/users/<username>/notebooks/<uuid>/
     return os.path.join(user_root(username), notebook_id)
 
-def ensure_tree(username: str, notebook_id: str):
 
+def ensure_tree(username: str, notebook_id: str):
+    # Ensure notebook folder layout exists
     base = nb_root(username, notebook_id)
 
+    os.makedirs(user_root(username), exist_ok=True)
     os.makedirs(base, exist_ok=True)
+
     os.makedirs(os.path.join(base, "files_raw"), exist_ok=True)
     os.makedirs(os.path.join(base, "files_extracted"), exist_ok=True)
-    os.makedirs(os.path.join(base, "artifacts"), exist_ok=True)
+    os.makedirs(os.path.join(base, "chroma"), exist_ok=True)
+
+    os.makedirs(os.path.join(base, "chat"), exist_ok=True)
+    os.makedirs(os.path.join(base, "artifacts", "reports"), exist_ok=True)
+    os.makedirs(os.path.join(base, "artifacts", "quizzes"), exist_ok=True)
+    os.makedirs(os.path.join(base, "artifacts", "podcasts"), exist_ok=True)
+
+    # Make sure index.json exists
+    ip = index_path(username)
+    if not os.path.exists(ip):
+        with open(ip, "w", encoding="utf-8") as f:
+            f.write("[]")