Upload folder via script

environment.py

@@ -10,8 +10,11 @@ VELAI_STORAGE_SECRET = os.getenv("VELAI_STORAGE_SECRET") or "super-secure"
 VELAI_ADMIN_PASSWORD = os.getenv("VELAI_ADMIN_PASSWORD") or "velai-admin"
 VELAI_APP_PASSWORD = os.getenv("VELAI_APP_PASSWORD") or ""
 
-VELAI_STORAGE_PATH = Path(os.getenv("VELAI_STORAGE_PATH") or ".storage")
-VELAI_BLOB_STORAGE_PATH = Path(os.getenv("VELAI_STORAGE_PATH") or VELAI_STORAGE_PATH / "blob")
+_velai_storage_path = Path(os.getenv("VELAI_STORAGE_PATH") or ".storage")
+VELAI_STORAGE_PATH = _velai_storage_path
+# When VELAI_STORAGE_PATH is set (e.g. /data on Hugging Face), blobs live alongside NiceGUI JSON files.
+_default_blob_path = _velai_storage_path if os.getenv("VELAI_STORAGE_PATH") else _velai_storage_path / "blob"
+VELAI_BLOB_STORAGE_PATH = Path(os.getenv("VELAI_BLOB_STORAGE_PATH") or _default_blob_path)
 
 VELAI_ENABLE_DUMMY = bool(os.getenv("VELAI_ENABLE_DUMMY")) or False
 VELAI_LOG_LEVEL = (os.getenv("VELAI_LOG_LEVEL") or "INFO").upper()

velai/app_context.py

@@ -12,7 +12,11 @@ from velai.context_registry import ContextRegistry
 from velai.storage.async_blob_storage import AsyncBlobStorageAdapter
 from velai.storage.blob_storage import BlobStorage, FileSystemBlobStorage
 from velai.storage.document_storage import DocumentStorage
-from velai.storage.keyvalue_storage import NiceGuiKeyValueStorage, NiceGuiStorageType
+from velai.storage.keyvalue_storage import (
+    FilePersistentDictStorage,
+    NiceGuiKeyValueStorage,
+    NiceGuiStorageType,
+)
 from velai.sync_utils import sync_wrap
 from velai.user_info import UserInfo
 
@@ -64,9 +68,24 @@ def _create_user_info(storage: DocumentStorage) -> UserInfo:
     return info
 
 
+def _document_storage_for_session(session_id: str, storage_type: NiceGuiStorageType) -> DocumentStorage:
+    """Load persisted NiceGUI storage from disk for plain HTTP handlers."""
+    from nicegui.persistence import FilePersistentDict
+
+    suffix = "general" if storage_type == NiceGuiStorageType.General else f"user-{session_id}"
+    path = environment.VELAI_STORAGE_PATH / f"storage-{suffix}.json"
+    persistent = FilePersistentDict(path, encoding="utf-8")
+    persistent.initialize_sync()
+    return DocumentStorage(FilePersistentDictStorage[dict[str, Any]](persistent))
+
+
 def _build_app_context(session_id: str, client: Client | None) -> AppContext:
-    app_storage = DocumentStorage(NiceGuiKeyValueStorage[dict[str, Any]](NiceGuiStorageType.General))
-    user_storage = DocumentStorage(NiceGuiKeyValueStorage[dict[str, Any]](NiceGuiStorageType.User))
+    if client is None:
+        app_storage = _document_storage_for_session(session_id, NiceGuiStorageType.General)
+        user_storage = _document_storage_for_session(session_id, NiceGuiStorageType.User)
+    else:
+        app_storage = DocumentStorage(NiceGuiKeyValueStorage[dict[str, Any]](NiceGuiStorageType.General))
+        user_storage = DocumentStorage(NiceGuiKeyValueStorage[dict[str, Any]](NiceGuiStorageType.User))
     blob_storage: BlobStorage = FileSystemBlobStorage(
         user_id=session_id, root_dir=environment.VELAI_BLOB_STORAGE_PATH, document_storage=user_storage
     )

velai/storage/async_blob_storage.py

@@ -117,14 +117,14 @@ class AsyncBlobStorageAdapter:
         )
         return AsyncBlob(self, blob.meta)
 
-    async def get(self, blob_id: str) -> AsyncBlob | None:
-        blob = await asyncio.to_thread(self._sync.get, blob_id)
+    async def get(self, blob_id: str, *, original_name: str | None = None) -> AsyncBlob | None:
+        blob = await asyncio.to_thread(self._sync.get, blob_id, original_name=original_name)
         if blob is None:
             return None
         return AsyncBlob(self, blob.meta)
 
-    async def require(self, blob_id: str) -> AsyncBlob:
-        blob = await self.get(blob_id)
+    async def require(self, blob_id: str, *, original_name: str | None = None) -> AsyncBlob:
+        blob = await self.get(blob_id, original_name=original_name)
         if blob is None:
             raise KeyError(f"Unknown blob_id: {blob_id}")
         return blob

velai/storage/blob_storage.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import hashlib
+import mimetypes
 import os
 import uuid
 from abc import ABC, abstractmethod
@@ -12,6 +13,8 @@ from .blob_models import Blob, BlobMeta
 from .document_storage import DocumentStorage
 from .utils import ensure_within_dir, safe_user_dir_name, sanitize_filename, utc_now
 
+ABSOLUTE_PATH_KEY = "absolute_path"
+
 
 class BlobStorage(ABC):
     @abstractmethod
@@ -103,6 +106,64 @@ class FileSystemBlobStorage(BlobStorage):
         ensure_within_dir(path, user_dir)
         return path
 
+    def _path_from_meta(self, meta: BlobMeta) -> Path:
+        abs_path = meta.extra.get(ABSOLUTE_PATH_KEY)
+        if abs_path:
+            return Path(str(abs_path))
+        return self._path_for_storage_name(meta.storage_name)
+
+    @staticmethod
+    def storage_name_for(blob_id: str, original_name: str) -> str:
+        original_name_s = sanitize_filename(original_name)
+        return sanitize_filename(f"{blob_id}_{original_name_s}", max_len=180)
+
+    def _blob_from_file(
+        self,
+        *,
+        blob_id: str,
+        original_name: str,
+        storage_name: str,
+        path: Path,
+    ) -> Blob:
+        stat = path.stat()
+        mime_type, _ = mimetypes.guess_type(original_name)
+        meta = BlobMeta(
+            blob_id=blob_id,
+            storage_name=storage_name,
+            original_name=sanitize_filename(original_name),
+            mime_type=mime_type,
+            size_bytes=stat.st_size,
+            sha256=None,
+            created_at=utc_now(),
+            updated_at=utc_now(),
+            extra={ABSOLUTE_PATH_KEY: str(path.resolve())},
+        )
+        return Blob(self, meta)
+
+    def get_by_filename(self, blob_id: str, original_name: str) -> Blob | None:
+        """Load a blob from disk when the index is missing or the session changed."""
+        storage_name = self.storage_name_for(blob_id, original_name)
+
+        path = self._path_for_storage_name(storage_name)
+        if path.is_file():
+            return self._blob_from_file(
+                blob_id=blob_id, original_name=original_name, storage_name=storage_name, path=path
+            )
+
+        for user_dir in self._root_dir.iterdir():
+            if not user_dir.is_dir():
+                continue
+            candidate = user_dir / storage_name
+            try:
+                ensure_within_dir(candidate, user_dir)
+            except ValueError:
+                continue
+            if candidate.is_file():
+                return self._blob_from_file(
+                    blob_id=blob_id, original_name=original_name, storage_name=storage_name, path=candidate
+                )
+        return None
+
     def _get_meta(self, blob_id: str) -> BlobMeta | None:
         index = self._load_index()
         blobs = self._get_blob_dict(index)
@@ -123,7 +184,7 @@ class FileSystemBlobStorage(BlobStorage):
     ) -> Blob:
         original_name_s = sanitize_filename(original_name)
         blob_id = str(uuid.uuid4())
-        storage_name = sanitize_filename(f"{blob_id}_{original_name_s}", max_len=180)
+        storage_name = self.storage_name_for(blob_id, original_name)
 
         sha256 = hashlib.sha256(data).hexdigest()
         size_bytes = len(data)
@@ -167,11 +228,13 @@ class FileSystemBlobStorage(BlobStorage):
 
         return Blob(self, meta)
 
-    def get(self, blob_id: str) -> Blob | None:
+    def get(self, blob_id: str, *, original_name: str | None = None) -> Blob | None:
         meta = self._get_meta(blob_id)
-        if meta is None:
-            return None
-        return Blob(self, meta)
+        if meta is not None:
+            return Blob(self, meta)
+        if original_name is not None:
+            return self.get_by_filename(blob_id, original_name)
+        return None
 
     def list(self) -> list[Blob]:
         index = self._load_index()
@@ -186,12 +249,10 @@ class FileSystemBlobStorage(BlobStorage):
         return [Blob(self, m) for m in items]
 
     def _open_by_meta(self, meta: BlobMeta) -> BinaryIO:
-        path = self._path_for_storage_name(meta.storage_name)
-        return open(path, "rb")
+        return open(self._path_from_meta(meta), "rb")
 
     def _read_bytes_by_meta(self, meta: BlobMeta) -> bytes:
-        path = self._path_for_storage_name(meta.storage_name)
-        with open(path, "rb") as f:
+        with open(self._path_from_meta(meta), "rb") as f:
             return f.read()
 
     def update_meta(

velai/storage/keyvalue_storage.py

@@ -95,3 +95,25 @@ class InMemoryKeyValueStorage(KeyValueStorage[T]):
         if prefix is None:
             return list(self._data.keys())
         return [k for k in self._data.keys() if k.startswith(prefix)]
+
+
+class FilePersistentDictStorage(KeyValueStorage[T]):
+    """Read NiceGUI on-disk storage files directly (for stateless HTTP handlers)."""
+
+    def __init__(self, persistent_dict) -> None:
+        self._persistent = persistent_dict
+
+    def get(self, key: str) -> T | None:
+        return cast(T | None, self._persistent.get(key))
+
+    def set(self, key: str, value: T) -> None:
+        self._persistent[key] = value
+
+    def delete(self, key: str) -> None:
+        self._persistent.pop(key, None)
+
+    def keys(self, prefix: str | None = None) -> list[str]:
+        all_keys = list(self._persistent.keys())
+        if prefix is None:
+            return all_keys
+        return [k for k in all_keys if k.startswith(prefix)]

velai/storage/storage_endpoint.py

@@ -68,7 +68,7 @@ def register():
         ctx = await app_context.app_context_from_request(request)
 
         try:
-            blob = await ctx.blob_storage.require(blob_id)
+            blob = await ctx.blob_storage.require(blob_id, original_name=original_name)
         except KeyError as exc:
             raise HTTPException(status_code=404, detail="blob not found") from exc