new test on storage management

index.html

@@ -1592,11 +1592,11 @@
       <div class="admin-row">
         <label>
           Type confirmation phrase
-          <input type="text" id="adminDeleteConfirmInput" placeholder="DELETE /data">
+          <input type="text" id="adminDeleteConfirmInput" placeholder="DELETE APP DATA">
         </label>
       </div>
       <div class="admin-actions">
-        <button type="button" id="wipeDataBtn" class="danger-btn">Delete all /data contents</button>
+        <button type="button" id="wipeDataBtn" class="danger-btn">Delete all app data contents</button>
       </div>
     </div>
     <div id="adminPanelStatus"></div>
@@ -3626,7 +3626,7 @@
         }
 
         const proceed = window.confirm(
-          'This will permanently delete all files/folders inside /data. Continue?'
+          'This will permanently delete all files/folders inside the app data directories. Continue?'
         );
         if (!proceed) {
           return;
@@ -3644,12 +3644,12 @@
 
           const data = await response.json();
           if (!response.ok || !data.ok) {
-            setAdminStatus(data.message || 'Failed to delete /data contents.', true);
+            setAdminStatus(data.message || 'Failed to delete app data contents.', true);
             return;
           }
 
           setAdminStatus(
-            `Deleted /data contents (files: ${data.deleted_files ?? 0}, folders: ${data.deleted_dirs ?? 0}).`
+            `Deleted app data contents (files: ${data.deleted_files ?? 0}, folders: ${data.deleted_dirs ?? 0}).`
           );
           adminDeleteConfirmInput.value = '';
         } catch (error) {

main.py

@@ -50,7 +50,7 @@ from pdf_text_processing import (
     get_pdf_title,
     entity_match
 )
-from vectorstore import get_vectorstore, compute_file_hash
+from vectorstore import get_vectorstore, compute_file_hash, CHROMA_DB_BASE
 from system_chart_extraction import system_chart_extraction
 from system_chart_service import SystemChartService
 from hypotheses_service import HypothesesService
@@ -304,17 +304,37 @@ def resource_path(relative_path: str) -> str:
         return os.path.join(sys._MEIPASS, relative_path)
     return os.path.join(os.path.dirname(__file__), relative_path)
 
-def get_base_directory():
-    """Return base directory, preferring persistent cloud storage when provided."""
-    persistent_dir = os.environ.get("PERSISTENT_STORAGE_PATH")
-    if persistent_dir:
-        return persistent_dir
-    if hasattr(sys, "_MEIPASS"):  # Running as PyInstaller executable
-        return os.path.dirname(sys.executable)
-    return os.path.dirname(os.path.abspath(__file__))
+def get_base_directory() -> str:
+    """
+    HF Spaces-safe data root:
+    - Prefer explicit override via APP_DATA_DIR or PERSISTENT_STORAGE_PATH
+    - Else use /data/<app> only if it exists and is writable
+    - Else use $HOME/.cache/<app>
+    - Else use /tmp/<app>
+    """
+    override = (os.environ.get("APP_DATA_DIR") or os.environ.get("PERSISTENT_STORAGE_PATH") or "").strip()
+    if override:
+        os.makedirs(override, exist_ok=True)
+        return override
+
+    if os.path.isdir("/data") and os.access("/data", os.W_OK):
+        base = "/data/pdf-rag"
+        os.makedirs(base, exist_ok=True)
+        return base
+
+    home = os.environ.get("HOME")
+    if home:
+        base = os.path.join(home, ".cache", "pdf-rag")
+        os.makedirs(base, exist_ok=True)
+        return base
+
+    base = "/tmp/pdf-rag"
+    os.makedirs(base, exist_ok=True)
+    return base
 
 # Define paths relative to the base directory for persistent storage
-UPLOAD_DIR = os.path.join(get_base_directory(), "uploaded files")
+APP_DATA_ROOT = get_base_directory()
+UPLOAD_DIR = os.path.join(APP_DATA_ROOT, "uploaded_files")
 os.makedirs(UPLOAD_DIR, exist_ok=True)
 logger.info(f"UPLOAD_DIR set to: {os.path.abspath(UPLOAD_DIR)}")
 
@@ -367,7 +387,7 @@ def load_existing_databases():
     
     # Get all database directories from root databases
     database_dirs = []
-    chroma_db_base = os.path.join(get_base_directory(), "root databases")
+    chroma_db_base = CHROMA_DB_BASE
     
     if os.path.exists(chroma_db_base):
         for item in os.listdir(chroma_db_base):
@@ -647,8 +667,7 @@ embedding_model = None
 llm_QA = None
 llm_system_chart_extraction = None
 
-ADMIN_DELETE_CONFIRMATION_TEXT = "DELETE /data"
-DEFAULT_ADMIN_PURGE_PATH = "/data"
+ADMIN_DELETE_CONFIRMATION_TEXT = "DELETE APP DATA"
 
 def get_model_provider(model: str) -> Optional[str]:
     if model in OPENAI_MODELS:
@@ -2890,24 +2909,29 @@ def get_admin_password() -> Optional[str]:
         return password.strip()
     return None
 
-def get_admin_purge_root() -> str:
-    configured = (os.environ.get("ADMIN_PURGE_PATH") or DEFAULT_ADMIN_PURGE_PATH).strip()
-    if not configured:
-        configured = DEFAULT_ADMIN_PURGE_PATH
-    return os.path.abspath(configured)
+def get_admin_wipe_targets() -> List[str]:
+    return [
+        os.path.abspath(UPLOAD_DIR),
+        os.path.abspath(CHROMA_DB_BASE),
+    ]
+
+def is_safe_target(path: str, data_root: str) -> bool:
+    normalized_path = os.path.abspath(path)
+    normalized_root = os.path.abspath(data_root)
 
-def is_safe_admin_purge_root(path: str) -> bool:
-    normalized = os.path.abspath(path)
-    # Must be an absolute subpath, never root.
-    if normalized in {"/", ""}:
+    if normalized_path in {"/", ""}:
         return False
-    if not normalized.startswith("/data"):
+
+    try:
+        return os.path.commonpath([normalized_root, normalized_path]) == normalized_root
+    except ValueError:
         return False
-    return True
 
 def delete_directory_contents(path: str) -> Dict[str, int]:
     deleted_files = 0
     deleted_dirs = 0
+    if not os.path.isdir(path):
+        return {"deleted_files": 0, "deleted_dirs": 0}
     for name in os.listdir(path):
         target = os.path.join(path, name)
         if os.path.isfile(target) or os.path.islink(target):
@@ -2962,39 +2986,54 @@ async def admin_wipe_data(
             status_code=400
         )
 
-    purge_root = get_admin_purge_root()
-    if not is_safe_admin_purge_root(purge_root):
-        return JSONResponse(
-            content={"ok": False, "message": f"Unsafe ADMIN_PURGE_PATH resolved to '{purge_root}'."},
-            status_code=400
-        )
-
-    if not os.path.exists(purge_root):
-        return JSONResponse(
-            content={"ok": False, "message": f"Purge path does not exist: {purge_root}"},
-            status_code=404
-        )
+    targets = get_admin_wipe_targets()
+    data_root = os.path.abspath(APP_DATA_ROOT)
 
-    if not os.path.isdir(purge_root):
-        return JSONResponse(
-            content={"ok": False, "message": f"Purge path is not a directory: {purge_root}"},
-            status_code=400
-        )
+    for target in targets:
+        if not is_safe_target(target, data_root):
+            return JSONResponse(
+                content={
+                    "ok": False,
+                    "message": f"Refusing to wipe unsafe target: {target}",
+                    "data_root": data_root,
+                    "targets": targets
+                },
+                status_code=400
+            )
 
     try:
-        result = delete_directory_contents(purge_root)
+        results = []
+        total_files = 0
+        total_dirs = 0
+
+        for target in targets:
+            if not os.path.exists(target):
+                results.append({"target": target, "status": "missing"})
+                continue
+
+            result = delete_directory_contents(target)
+            total_files += result["deleted_files"]
+            total_dirs += result["deleted_dirs"]
+            results.append({"target": target, "status": "wiped", **result})
+
+        FILE_REGISTRY.clear()
+        EMBEDDING_PROGRESS.clear()
+        HYPOTHESIS_GENERATION_CONTEXT.clear()
+
         logger.warning(
-            "Admin data wipe executed for %s (files=%s, dirs=%s)",
-            purge_root,
-            result["deleted_files"],
-            result["deleted_dirs"]
+            "Admin data wipe executed for targets=%s (files=%s, dirs=%s)",
+            targets,
+            total_files,
+            total_dirs
         )
         return JSONResponse(
             content={
                 "ok": True,
-                "message": f"Deleted contents of {purge_root}.",
-                "deleted_files": result["deleted_files"],
-                "deleted_dirs": result["deleted_dirs"]
+                "message": "Wipe completed.",
+                "data_root": data_root,
+                "targets": results,
+                "deleted_files": total_files,
+                "deleted_dirs": total_dirs
             },
             status_code=200
         )
@@ -3216,7 +3255,6 @@ if __name__ == "__main__":
         logger.info(f"Executable path: {sys.executable}")
     logger.info(f"Base directory for storage: {get_base_directory()}")
     logger.info(f"UPLOAD_DIR: {os.path.abspath(UPLOAD_DIR)}")
-    from vectorstore import CHROMA_DB_BASE
     logger.info(f"CHROMA_DB_BASE: {os.path.abspath(CHROMA_DB_BASE)}")
     logger.info("=" * 50)
        

vectorstore.py

@@ -1,15 +1,35 @@
 import os
-import sys
 import hashlib
 from langchain_community.vectorstores import Chroma
 from langchain_openai import OpenAIEmbeddings
 
-def get_base_directory():
-    """Return the base directory for persistent storage (next to executable when packaged)"""
-    if hasattr(sys, '_MEIPASS'):  # Running as PyInstaller executable
-        return os.path.dirname(sys.executable)
-    else:  # Running as normal Python script
-        return os.path.dirname(os.path.abspath(__file__))
+def get_base_directory() -> str:
+    """
+    HF Spaces-safe data root:
+    - Prefer explicit override via APP_DATA_DIR or PERSISTENT_STORAGE_PATH
+    - Else use /data/<app> only if it exists and is writable
+    - Else use $HOME/.cache/<app>
+    - Else use /tmp/<app>
+    """
+    override = (os.environ.get("APP_DATA_DIR") or os.environ.get("PERSISTENT_STORAGE_PATH") or "").strip()
+    if override:
+        os.makedirs(override, exist_ok=True)
+        return override
+
+    if os.path.isdir("/data") and os.access("/data", os.W_OK):
+        base = "/data/pdf-rag"
+        os.makedirs(base, exist_ok=True)
+        return base
+
+    home = os.environ.get("HOME")
+    if home:
+        base = os.path.join(home, ".cache", "pdf-rag")
+        os.makedirs(base, exist_ok=True)
+        return base
+
+    base = "/tmp/pdf-rag"
+    os.makedirs(base, exist_ok=True)
+    return base
 
 # Define path relative to the base directory
 CHROMA_DB_BASE = os.path.join(get_base_directory(), "root databases")