"""Per-student persistence in a private Hugging Face Dataset repo. The container filesystem is ephemeral — it resets whenever the Space restarts — so anything a student saves has to leave the container. A private Dataset repo is free, unlimited, versioned, and has no inactivity pause, which suits a beta. Layout, one file per student: students/{student_id}/profile.json students/{student_id}/syllabi.json **One file per student, never a shared mutable file.** A dataset repo is git, not a database: there is no row locking, so two writers on one file either clobber each other or lose a race. Per-student files make that impossible between students, and `write_json` retries on the remaining same-student case. Every write is a network commit (hundreds of ms to seconds), so callers should debounce rather than save on every keystroke — see `app/store.js`. Local development: with no `HF_TOKEN` set, reads and writes fall back to a git-ignored `.data/` directory so the app runs with no credentials at all. """ from __future__ import annotations import json import logging import os import time from pathlib import Path log = logging.getLogger("foresight.storage") DATASET_REPO = os.environ.get("FORESIGHT_DATASET_REPO", "umangchaudhry/foresight") LOCAL_DIR = Path(os.environ.get("FORESIGHT_LOCAL_DATA_DIR", ".data")) WRITE_ATTEMPTS = 3 def token() -> str | None: """HF write token. Must be set as a *Space secret* for deployment — a GitHub Actions secret only covers pushing the code to the Space.""" return os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") def using_hub() -> bool: return bool(token()) def student_path(student_id: str, name: str) -> str: return f"students/{student_id}/{name}" # --- local fallback --------------------------------------------------------- def _local_read(path: str): f = LOCAL_DIR / path if not f.exists(): return None return json.loads(f.read_text(encoding="utf-8")) def _local_write(path: str, payload: dict) -> None: f = LOCAL_DIR / path f.parent.mkdir(parents=True, exist_ok=True) f.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8") # --- hub ------------------------------------------------------------------- def _hub_read(path: str): from huggingface_hub import hf_hub_download from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError try: local = hf_hub_download( repo_id=DATASET_REPO, filename=path, repo_type="dataset", token=token(), force_download=True, ) except (EntryNotFoundError, RepositoryNotFoundError): return None except Exception as err: # 404s surface in several shapes if "404" in str(err) or "not found" in str(err).lower(): return None raise return json.loads(Path(local).read_text(encoding="utf-8")) def _hub_write(path: str, payload: dict, message: str) -> None: from huggingface_hub import HfApi body = json.dumps(payload, indent=2, ensure_ascii=False).encode("utf-8") api = HfApi(token=token()) last: Exception | None = None for attempt in range(1, WRITE_ATTEMPTS + 1): try: api.upload_file( path_or_fileobj=body, path_in_repo=path, repo_id=DATASET_REPO, repo_type="dataset", commit_message=message, ) return except Exception as err: # A concurrent commit to the same file loses the race; retry on a # fresh head rather than dropping the student's save. last = err if attempt < WRITE_ATTEMPTS: time.sleep(0.5 * attempt) raise RuntimeError(f"failed to write {path} after {WRITE_ATTEMPTS} attempts: {last}") # --- public API ------------------------------------------------------------ def read_json(path: str): """Stored JSON at `path`, or None if it doesn't exist yet.""" if using_hub(): return _hub_read(path) return _local_read(path) def write_json(path: str, payload: dict, message: str | None = None) -> None: if using_hub(): _hub_write(path, payload, message or f"update {path}") else: log.warning("HF_TOKEN unset — writing %s to %s instead of the dataset repo", path, LOCAL_DIR) _local_write(path, payload) def describe() -> dict: """Where data is going, for /healthz and startup logging.""" return { "backend": "huggingface-dataset" if using_hub() else "local-directory", "dataset_repo": DATASET_REPO if using_hub() else None, "local_dir": None if using_hub() else str(LOCAL_DIR), }