Spaces:
Running
Running
File size: 4,729 Bytes
6b7e4ba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | """Per-student persistence in a private Hugging Face Dataset repo.
The container filesystem is ephemeral — it resets whenever the Space restarts —
so anything a student saves has to leave the container. A private Dataset repo is
free, unlimited, versioned, and has no inactivity pause, which suits a beta.
Layout, one file per student:
students/{student_id}/profile.json
students/{student_id}/syllabi.json
**One file per student, never a shared mutable file.** A dataset repo is git, not
a database: there is no row locking, so two writers on one file either clobber
each other or lose a race. Per-student files make that impossible between
students, and `write_json` retries on the remaining same-student case.
Every write is a network commit (hundreds of ms to seconds), so callers should
debounce rather than save on every keystroke — see `app/store.js`.
Local development: with no `HF_TOKEN` set, reads and writes fall back to a
git-ignored `.data/` directory so the app runs with no credentials at all.
"""
from __future__ import annotations
import json
import logging
import os
import time
from pathlib import Path
log = logging.getLogger("foresight.storage")
DATASET_REPO = os.environ.get("FORESIGHT_DATASET_REPO", "umangchaudhry/foresight")
LOCAL_DIR = Path(os.environ.get("FORESIGHT_LOCAL_DATA_DIR", ".data"))
WRITE_ATTEMPTS = 3
def token() -> str | None:
"""HF write token. Must be set as a *Space secret* for deployment —
a GitHub Actions secret only covers pushing the code to the Space."""
return os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN")
def using_hub() -> bool:
return bool(token())
def student_path(student_id: str, name: str) -> str:
return f"students/{student_id}/{name}"
# --- local fallback ---------------------------------------------------------
def _local_read(path: str):
f = LOCAL_DIR / path
if not f.exists():
return None
return json.loads(f.read_text(encoding="utf-8"))
def _local_write(path: str, payload: dict) -> None:
f = LOCAL_DIR / path
f.parent.mkdir(parents=True, exist_ok=True)
f.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
# --- hub -------------------------------------------------------------------
def _hub_read(path: str):
from huggingface_hub import hf_hub_download
from huggingface_hub.errors import EntryNotFoundError, RepositoryNotFoundError
try:
local = hf_hub_download(
repo_id=DATASET_REPO, filename=path, repo_type="dataset",
token=token(), force_download=True,
)
except (EntryNotFoundError, RepositoryNotFoundError):
return None
except Exception as err: # 404s surface in several shapes
if "404" in str(err) or "not found" in str(err).lower():
return None
raise
return json.loads(Path(local).read_text(encoding="utf-8"))
def _hub_write(path: str, payload: dict, message: str) -> None:
from huggingface_hub import HfApi
body = json.dumps(payload, indent=2, ensure_ascii=False).encode("utf-8")
api = HfApi(token=token())
last: Exception | None = None
for attempt in range(1, WRITE_ATTEMPTS + 1):
try:
api.upload_file(
path_or_fileobj=body, path_in_repo=path,
repo_id=DATASET_REPO, repo_type="dataset",
commit_message=message,
)
return
except Exception as err:
# A concurrent commit to the same file loses the race; retry on a
# fresh head rather than dropping the student's save.
last = err
if attempt < WRITE_ATTEMPTS:
time.sleep(0.5 * attempt)
raise RuntimeError(f"failed to write {path} after {WRITE_ATTEMPTS} attempts: {last}")
# --- public API ------------------------------------------------------------
def read_json(path: str):
"""Stored JSON at `path`, or None if it doesn't exist yet."""
if using_hub():
return _hub_read(path)
return _local_read(path)
def write_json(path: str, payload: dict, message: str | None = None) -> None:
if using_hub():
_hub_write(path, payload, message or f"update {path}")
else:
log.warning("HF_TOKEN unset — writing %s to %s instead of the dataset repo", path, LOCAL_DIR)
_local_write(path, payload)
def describe() -> dict:
"""Where data is going, for /healthz and startup logging."""
return {
"backend": "huggingface-dataset" if using_hub() else "local-directory",
"dataset_repo": DATASET_REPO if using_hub() else None,
"local_dir": None if using_hub() else str(LOCAL_DIR),
}
|