Spaces:
Sleeping
Sleeping
Download hub.py from BVendeville/reading-study-validity: direct link, hf CLI and curl.
- Browser
- Download file 15.6 kB
-
https://huggingface.co/spaces/BVendeville/reading-study-validity/resolve/main/hub.py
- Command line
-
hf download hf://spaces/BVendeville/reading-study-validity/hub.py
-
curl -L -o hub.py https://huggingface.co/spaces/BVendeville/reading-study-validity/resolve/main/hub.py
15.6 kB
| """Keep the study's private files in a PRIVATE Hugging Face dataset repo, for a public Space. | |
| A public Space's files are readable by anyone, so the annotator codes (config) and the documents (which name the | |
| system behind every text) cannot live there; and a Space's disk is wiped on every restart, rebuild or sleep, so | |
| answers cannot live only there either. With ``ANNOTATION_HUB_REPO`` set, the app: | |
| 1. downloads the config and the documents from that repo at startup (``ANNOTATION_CONFIG``, ``ANNOTATION_DATA`` | |
| are then paths inside the repo); | |
| 2. restores the existing answer files from it before serving, so resume works after any restart; | |
| 3. after every completed document, pushes that annotator's whole answer file back to it, from a background thread; | |
| files queued within a minute go up together in one commit (the Hub limits commits per hour). | |
| The local write (append + fsync, core.append_record) still happens first and is what the annotator waits for. The | |
| push follows within a minute; failures are retried with backoff and re-queued, and pending pushes are flushed on | |
| shutdown without waiting for the minute. Repo layout: ``config.yaml``, ``data/<name>.json``, | |
| ``annotations/<data stem>/<code>.jsonl``. | |
| The ``Store`` interface is small so tests can swap in ``LocalStore`` (a directory standing in for the repo). | |
| """ | |
| from __future__ import annotations | |
| import fcntl | |
| import os | |
| import queue | |
| import shutil | |
| import tempfile | |
| import threading | |
| import time | |
| from pathlib import Path | |
| import core as C | |
| REPO_TYPE = "dataset" | |
| class LocalStore: | |
| """A directory standing in for the private repo (tests, and dry runs without network).""" | |
| def __init__(self, root: str | Path): | |
| self.root = Path(root) | |
| def download(self, path_in_repo: str, dest: Path) -> Path | None: | |
| src = self.root / path_in_repo | |
| if not src.exists(): | |
| return None | |
| dest.parent.mkdir(parents=True, exist_ok=True) | |
| shutil.copyfile(src, dest) | |
| return dest | |
| def list(self, prefix: str) -> list[str]: | |
| base = self.root / prefix | |
| return sorted(str(p.relative_to(self.root)) for p in base.rglob("*") if p.is_file()) if base.exists() else [] | |
| def upload(self, data: bytes, path_in_repo: str, message: str) -> None: | |
| dest = self.root / path_in_repo | |
| dest.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = dest.with_suffix(dest.suffix + ".tmp") | |
| tmp.write_bytes(data) | |
| os.replace(tmp, dest) | |
| def upload_many(self, files: dict[str, bytes], message: str) -> None: | |
| for path_in_repo, data in files.items(): | |
| self.upload(data, path_in_repo, message) | |
| class HubStore: | |
| """The private dataset repo on the Hugging Face Hub.""" | |
| def __init__(self, repo_id: str, token: str | None = None): | |
| from huggingface_hub import HfApi | |
| self.repo_id = repo_id | |
| self.token = token or os.environ.get("HF_TOKEN") or None | |
| if not self.token and os.environ.get("SPACE_ID"): | |
| raise RuntimeError("HF_TOKEN is not set: add a fine-grained token with read+write access to " | |
| f"{repo_id} as the Space secret HF_TOKEN (make_space.py set-token), then restart") | |
| self.api = HfApi(token=self.token) | |
| info = self.api.repo_info(repo_id, repo_type=REPO_TYPE) # fails loudly now (missing repo, bad token) | |
| if not getattr(info, "private", True): | |
| raise RuntimeError(f"{repo_id} is PUBLIC: it would expose the annotator codes and the system names. " | |
| "Make it private first.") | |
| def download(self, path_in_repo: str, dest: Path) -> Path | None: | |
| """None ONLY when the file does not exist in the repo. Any other failure (network, 5xx, 429) raises: | |
| huggingface_hub reports those as LocalEntryNotFoundError, a subclass of EntryNotFoundError, and treating | |
| them as 'no file' once let a transient outage pass for 'no answers yet'.""" | |
| from huggingface_hub import hf_hub_download | |
| from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError | |
| tmp = tempfile.mkdtemp(prefix="annot_dl_") | |
| try: | |
| try: | |
| got = hf_hub_download(self.repo_id, path_in_repo, repo_type=REPO_TYPE, token=self.token, | |
| local_dir=tmp) | |
| except LocalEntryNotFoundError: | |
| raise | |
| except EntryNotFoundError: | |
| return None | |
| dest.parent.mkdir(parents=True, exist_ok=True) | |
| shutil.move(got, dest) | |
| return dest | |
| finally: | |
| shutil.rmtree(tmp, ignore_errors=True) # every push downloads: do not leave a dir behind each time | |
| def list(self, prefix: str) -> list[str]: | |
| prefix = prefix.rstrip("/") + "/" | |
| return sorted(f for f in self.api.list_repo_files(self.repo_id, repo_type=REPO_TYPE) if f.startswith(prefix)) | |
| def upload(self, data: bytes, path_in_repo: str, message: str) -> None: | |
| self.api.upload_file(path_or_fileobj=data, path_in_repo=path_in_repo, repo_id=self.repo_id, | |
| repo_type=REPO_TYPE, commit_message=message) | |
| def upload_many(self, files: dict[str, bytes], message: str) -> None: | |
| """Several files in ONE commit.""" | |
| from huggingface_hub import CommitOperationAdd | |
| self.api.create_commit(self.repo_id, repo_type=REPO_TYPE, commit_message=message, | |
| operations=[CommitOperationAdd(path_in_repo=k, path_or_fileobj=v) for k, v in files.items()]) | |
| def open_store(repo: str): | |
| """``file:///some/dir`` = a local directory standing in for the repo (dry runs, tests); else a Hub repo id.""" | |
| return LocalStore(repo[len("file://"):]) if repo.startswith("file://") else HubStore(repo) | |
| def fetch(store, path_in_repo: str, cache_dir: Path) -> Path: | |
| """Download one required file (config, documents) into a private cache dir outside the app folder.""" | |
| got = store.download(path_in_repo, Path(cache_dir) / path_in_repo) | |
| if got is None: | |
| raise FileNotFoundError(f"{path_in_repo} not found in the private repo") | |
| return got | |
| def merge_lines(local: list[dict], remote: list[dict]) -> list[dict]: | |
| """Union of two answer files by doc_id. The local line wins, then remote-only lines in their order.""" | |
| seen = {r.get("doc_id") for r in local} | |
| return local + [r for r in remote if r.get("doc_id") not in seen] | |
| class AnswerSync: | |
| """Restores answer files at startup, re-reads an annotator's file from the repo when they log in, and pushes it | |
| after each completed document. | |
| Invariant: the repo copy only ever grows. Every push first downloads the repo copy, merges it with the local | |
| file (union by doc_id, under the file's exclusive lock, written back locally) and uploads the union; any error | |
| reading the repo copy aborts that attempt (it is retried) instead of being read as "no answers". | |
| """ | |
| def __init__(self, store, out_dir: str | Path, prefix: str, push_local_only: bool = False, retries: int = 5, | |
| backoff: float = 2.0, min_interval: float = 60.0): | |
| self.store, self.out_dir, self.prefix = store, Path(out_dir), prefix.rstrip("/") | |
| self.push_local_only = push_local_only # True only for an explicit, persistent output dir | |
| self.retries, self.backoff = retries, backoff | |
| self.min_interval = min_interval # seconds between two commits (both Spaces share one account's quota) | |
| self._last_commit = float("-inf") # time.monotonic() of the last commit attempt | |
| self._wake = threading.Event() | |
| self._q: queue.Queue[str] = queue.Queue() | |
| self._pending: set[str] = set() | |
| self._lock = threading.Lock() | |
| self.failed: set[str] = set() # codes whose last push failed after all retries (re-queued later) | |
| self.pushed = 0 | |
| self._thread = threading.Thread(target=self._run, name="answer-sync", daemon=True) | |
| self._thread.start() | |
| # ------------------------------------------------------------------ reading the repo copy | |
| def _remote(self, code: str) -> list[dict]: | |
| """The repo's lines for this code; [] only if the repo has no file for it. Errors propagate.""" | |
| tmp = Path(tempfile.mkdtemp(prefix="annot_pull_")) | |
| try: | |
| got = self.store.download(f"{self.prefix}/{code}.jsonl", tmp / f"{code}.jsonl") | |
| return C._parse_lines(got.read_text(encoding="utf-8")) if got else [] | |
| finally: | |
| shutil.rmtree(tmp, ignore_errors=True) | |
| def _merge_into_local(self, code: str, remote: list[dict]) -> bytes: | |
| """Under the local file's exclusive lock: union with ``remote``, write back if it grew, return the bytes.""" | |
| import json | |
| p = C.output_path(self.out_dir, code) | |
| if not remote and not p.exists(): | |
| return b"" # e.g. an unknown ID typed at "I'm coming back": do not create an empty file for it | |
| p.parent.mkdir(parents=True, exist_ok=True) | |
| with open(p, "a+", encoding="utf-8") as f: | |
| fcntl.flock(f, fcntl.LOCK_EX) | |
| try: | |
| f.seek(0) | |
| local = C._parse_lines(f.read()) | |
| merged = merge_lines(local, remote) | |
| data = "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in merged) | |
| if len(merged) != len(local): | |
| f.seek(0) | |
| f.truncate() | |
| f.write(data) | |
| f.flush() | |
| os.fsync(f.fileno()) | |
| return data.encode("utf-8") | |
| finally: | |
| fcntl.flock(f, fcntl.LOCK_UN) | |
| def pull(self, code: str) -> int: | |
| """Merge the repo's copy of this annotator's file into the local one (at login, so a second process or an | |
| earlier container cannot make them redo documents). Returns the number of lines; raises on Hub errors.""" | |
| data = self._merge_into_local(code, self._remote(code)) | |
| return sum(1 for line in data.splitlines() if line.strip()) | |
| def remote_has(self, code: str) -> bool: | |
| """Whether the repo already has a file for this ID (another process may have signed it up).""" | |
| return f"{self.prefix}/{code}.jsonl" in self.store.list(self.prefix) | |
| # ------------------------------------------------------------------ startup | |
| def restore(self, attempts: int = 4) -> dict[str, int]: | |
| """Bring every remote answer file into out_dir before the app serves. A Hub error is retried, then raised: | |
| better a Space that fails to start (and says why) than one that serves without the answers.""" | |
| self.out_dir.mkdir(parents=True, exist_ok=True) | |
| for attempt in range(attempts): | |
| try: | |
| counts = {} | |
| remote_codes = [Path(p).name[:-len(".jsonl")] for p in self.store.list(self.prefix) | |
| if p.endswith(".jsonl")] | |
| for code in remote_codes: | |
| remote = self._remote(code) | |
| data = self._merge_into_local(code, remote) | |
| counts[code] = sum(1 for line in data.splitlines() if line.strip()) | |
| if counts[code] != len(remote): | |
| self.push(code) | |
| break | |
| except Exception as e: # noqa: BLE001 | |
| if attempt == attempts - 1: | |
| raise RuntimeError(f"could not read the answers from the private repo: {e}") from e | |
| print(f"[answer-sync] restore failed ({e}); retrying", flush=True) | |
| time.sleep(self.backoff * 2 ** attempt) | |
| for p in self.out_dir.glob("*.jsonl"): # local-only files: pushed only from persistent storage | |
| if p.stem not in counts: | |
| counts[p.stem] = len(C.read_records(self.out_dir, p.stem)) | |
| if self.push_local_only: | |
| self.push(p.stem) | |
| else: | |
| print(f"[answer-sync] NOT pushing local-only {p.name}: the output dir was not set explicitly", | |
| flush=True) | |
| return counts | |
| # ------------------------------------------------------------------ pushing | |
| def push(self, code: str) -> None: | |
| """Queue this annotator's file for upload. Several pushes of one code before it is uploaded coalesce.""" | |
| with self._lock: | |
| if code in self._pending: | |
| return | |
| self._pending.add(code) | |
| self._q.put(code) | |
| def _requeue_failed(self) -> None: | |
| for code in list(self.failed): | |
| self.push(code) | |
| def _run(self) -> None: | |
| while True: | |
| try: | |
| first = self._q.get(timeout=60) | |
| except queue.Empty: # idle: retry anything that failed earlier | |
| self._requeue_failed() | |
| continue | |
| codes = [first] | |
| try: | |
| # One commit per interval, carrying every file queued meanwhile: the Hub has an hourly commit quota | |
| # (not documented; "exceeded our hourly quotas for action: commit"), and one commit per finished | |
| # document reaches it when several people work at once. flush() cuts the wait short. | |
| self._wake.wait(max(0.0, self._last_commit + self.min_interval - time.monotonic())) | |
| while True: | |
| try: | |
| codes.append(self._q.get_nowait()) | |
| except queue.Empty: | |
| break | |
| with self._lock: | |
| self._pending.difference_update(codes) # a write after this point re-queues the code | |
| self._push_batch(sorted(set(codes) | self.failed)) # earlier failures ride along | |
| finally: | |
| for _ in codes: | |
| self._q.task_done() | |
| def _push_batch(self, codes: list[str]) -> None: | |
| for attempt in range(self.retries): | |
| try: | |
| files = {f"{self.prefix}/{c}.jsonl": self._merge_into_local(c, self._remote(c)) # never shrink | |
| for c in codes} | |
| files = {k: v for k, v in files.items() if v} | |
| if files: | |
| self.store.upload_many(files, f"answers: {', '.join(codes)}") | |
| self._last_commit = time.monotonic() | |
| self.pushed += len(files) | |
| self.failed.difference_update(codes) | |
| return | |
| except Exception as e: # noqa: BLE001 -- network, rate limit, hub outage: retry, never crash | |
| print(f"[answer-sync] push of {', '.join(codes)} failed (attempt {attempt + 1}/{self.retries}): {e}", | |
| flush=True) | |
| time.sleep(self.backoff * 2 ** attempt) | |
| self._last_commit = time.monotonic() | |
| self.failed.update(codes) | |
| print(f"[answer-sync] giving up on {', '.join(codes)} for now; safe on local disk, retried with the next push " | |
| f"or within a minute", flush=True) | |
| def flush(self, timeout: float = 30.0) -> bool: | |
| """Wait until every queued push is done (or timeout). Also retries codes that failed earlier. For shutdown: | |
| from here on pushes no longer wait for the commit interval.""" | |
| self._wake.set() | |
| self._requeue_failed() | |
| t0 = time.time() | |
| while self._q.unfinished_tasks: | |
| if time.time() - t0 > timeout: | |
| return False | |
| time.sleep(0.1) | |
| return not self.failed | |