"""Keep the study's private files in a PRIVATE Hugging Face dataset repo, for a public Space. A public Space's files are readable by anyone, so the annotator codes (config) and the documents (which name the system behind every text) cannot live there; and a Space's disk is wiped on every restart, rebuild or sleep, so answers cannot live only there either. With ``ANNOTATION_HUB_REPO`` set, the app: 1. downloads the config and the documents from that repo at startup (``ANNOTATION_CONFIG``, ``ANNOTATION_DATA`` are then paths inside the repo); 2. restores the existing answer files from it before serving, so resume works after any restart; 3. after every completed document, pushes that annotator's whole answer file back to it, from a background thread; files queued within a minute go up together in one commit (the Hub limits commits per hour). The local write (append + fsync, core.append_record) still happens first and is what the annotator waits for. The push follows within a minute; failures are retried with backoff and re-queued, and pending pushes are flushed on shutdown without waiting for the minute. Repo layout: ``config.yaml``, ``data/.json``, ``annotations//.jsonl``. The ``Store`` interface is small so tests can swap in ``LocalStore`` (a directory standing in for the repo). """ from __future__ import annotations import fcntl import os import queue import shutil import tempfile import threading import time from pathlib import Path import core as C REPO_TYPE = "dataset" class LocalStore: """A directory standing in for the private repo (tests, and dry runs without network).""" def __init__(self, root: str | Path): self.root = Path(root) def download(self, path_in_repo: str, dest: Path) -> Path | None: src = self.root / path_in_repo if not src.exists(): return None dest.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(src, dest) return dest def list(self, prefix: str) -> list[str]: base = self.root / prefix return sorted(str(p.relative_to(self.root)) for p in base.rglob("*") if p.is_file()) if base.exists() else [] def upload(self, data: bytes, path_in_repo: str, message: str) -> None: dest = self.root / path_in_repo dest.parent.mkdir(parents=True, exist_ok=True) tmp = dest.with_suffix(dest.suffix + ".tmp") tmp.write_bytes(data) os.replace(tmp, dest) def upload_many(self, files: dict[str, bytes], message: str) -> None: for path_in_repo, data in files.items(): self.upload(data, path_in_repo, message) class HubStore: """The private dataset repo on the Hugging Face Hub.""" def __init__(self, repo_id: str, token: str | None = None): from huggingface_hub import HfApi self.repo_id = repo_id self.token = token or os.environ.get("HF_TOKEN") or None if not self.token and os.environ.get("SPACE_ID"): raise RuntimeError("HF_TOKEN is not set: add a fine-grained token with read+write access to " f"{repo_id} as the Space secret HF_TOKEN (make_space.py set-token), then restart") self.api = HfApi(token=self.token) info = self.api.repo_info(repo_id, repo_type=REPO_TYPE) # fails loudly now (missing repo, bad token) if not getattr(info, "private", True): raise RuntimeError(f"{repo_id} is PUBLIC: it would expose the annotator codes and the system names. " "Make it private first.") def download(self, path_in_repo: str, dest: Path) -> Path | None: """None ONLY when the file does not exist in the repo. Any other failure (network, 5xx, 429) raises: huggingface_hub reports those as LocalEntryNotFoundError, a subclass of EntryNotFoundError, and treating them as 'no file' once let a transient outage pass for 'no answers yet'.""" from huggingface_hub import hf_hub_download from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError tmp = tempfile.mkdtemp(prefix="annot_dl_") try: try: got = hf_hub_download(self.repo_id, path_in_repo, repo_type=REPO_TYPE, token=self.token, local_dir=tmp) except LocalEntryNotFoundError: raise except EntryNotFoundError: return None dest.parent.mkdir(parents=True, exist_ok=True) shutil.move(got, dest) return dest finally: shutil.rmtree(tmp, ignore_errors=True) # every push downloads: do not leave a dir behind each time def list(self, prefix: str) -> list[str]: prefix = prefix.rstrip("/") + "/" return sorted(f for f in self.api.list_repo_files(self.repo_id, repo_type=REPO_TYPE) if f.startswith(prefix)) def upload(self, data: bytes, path_in_repo: str, message: str) -> None: self.api.upload_file(path_or_fileobj=data, path_in_repo=path_in_repo, repo_id=self.repo_id, repo_type=REPO_TYPE, commit_message=message) def upload_many(self, files: dict[str, bytes], message: str) -> None: """Several files in ONE commit.""" from huggingface_hub import CommitOperationAdd self.api.create_commit(self.repo_id, repo_type=REPO_TYPE, commit_message=message, operations=[CommitOperationAdd(path_in_repo=k, path_or_fileobj=v) for k, v in files.items()]) def open_store(repo: str): """``file:///some/dir`` = a local directory standing in for the repo (dry runs, tests); else a Hub repo id.""" return LocalStore(repo[len("file://"):]) if repo.startswith("file://") else HubStore(repo) def fetch(store, path_in_repo: str, cache_dir: Path) -> Path: """Download one required file (config, documents) into a private cache dir outside the app folder.""" got = store.download(path_in_repo, Path(cache_dir) / path_in_repo) if got is None: raise FileNotFoundError(f"{path_in_repo} not found in the private repo") return got def merge_lines(local: list[dict], remote: list[dict]) -> list[dict]: """Union of two answer files by doc_id. The local line wins, then remote-only lines in their order.""" seen = {r.get("doc_id") for r in local} return local + [r for r in remote if r.get("doc_id") not in seen] class AnswerSync: """Restores answer files at startup, re-reads an annotator's file from the repo when they log in, and pushes it after each completed document. Invariant: the repo copy only ever grows. Every push first downloads the repo copy, merges it with the local file (union by doc_id, under the file's exclusive lock, written back locally) and uploads the union; any error reading the repo copy aborts that attempt (it is retried) instead of being read as "no answers". """ def __init__(self, store, out_dir: str | Path, prefix: str, push_local_only: bool = False, retries: int = 5, backoff: float = 2.0, min_interval: float = 60.0): self.store, self.out_dir, self.prefix = store, Path(out_dir), prefix.rstrip("/") self.push_local_only = push_local_only # True only for an explicit, persistent output dir self.retries, self.backoff = retries, backoff self.min_interval = min_interval # seconds between two commits (both Spaces share one account's quota) self._last_commit = float("-inf") # time.monotonic() of the last commit attempt self._wake = threading.Event() self._q: queue.Queue[str] = queue.Queue() self._pending: set[str] = set() self._lock = threading.Lock() self.failed: set[str] = set() # codes whose last push failed after all retries (re-queued later) self.pushed = 0 self._thread = threading.Thread(target=self._run, name="answer-sync", daemon=True) self._thread.start() # ------------------------------------------------------------------ reading the repo copy def _remote(self, code: str) -> list[dict]: """The repo's lines for this code; [] only if the repo has no file for it. Errors propagate.""" tmp = Path(tempfile.mkdtemp(prefix="annot_pull_")) try: got = self.store.download(f"{self.prefix}/{code}.jsonl", tmp / f"{code}.jsonl") return C._parse_lines(got.read_text(encoding="utf-8")) if got else [] finally: shutil.rmtree(tmp, ignore_errors=True) def _merge_into_local(self, code: str, remote: list[dict]) -> bytes: """Under the local file's exclusive lock: union with ``remote``, write back if it grew, return the bytes.""" import json p = C.output_path(self.out_dir, code) if not remote and not p.exists(): return b"" # e.g. an unknown ID typed at "I'm coming back": do not create an empty file for it p.parent.mkdir(parents=True, exist_ok=True) with open(p, "a+", encoding="utf-8") as f: fcntl.flock(f, fcntl.LOCK_EX) try: f.seek(0) local = C._parse_lines(f.read()) merged = merge_lines(local, remote) data = "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in merged) if len(merged) != len(local): f.seek(0) f.truncate() f.write(data) f.flush() os.fsync(f.fileno()) return data.encode("utf-8") finally: fcntl.flock(f, fcntl.LOCK_UN) def pull(self, code: str) -> int: """Merge the repo's copy of this annotator's file into the local one (at login, so a second process or an earlier container cannot make them redo documents). Returns the number of lines; raises on Hub errors.""" data = self._merge_into_local(code, self._remote(code)) return sum(1 for line in data.splitlines() if line.strip()) def remote_has(self, code: str) -> bool: """Whether the repo already has a file for this ID (another process may have signed it up).""" return f"{self.prefix}/{code}.jsonl" in self.store.list(self.prefix) # ------------------------------------------------------------------ startup def restore(self, attempts: int = 4) -> dict[str, int]: """Bring every remote answer file into out_dir before the app serves. A Hub error is retried, then raised: better a Space that fails to start (and says why) than one that serves without the answers.""" self.out_dir.mkdir(parents=True, exist_ok=True) for attempt in range(attempts): try: counts = {} remote_codes = [Path(p).name[:-len(".jsonl")] for p in self.store.list(self.prefix) if p.endswith(".jsonl")] for code in remote_codes: remote = self._remote(code) data = self._merge_into_local(code, remote) counts[code] = sum(1 for line in data.splitlines() if line.strip()) if counts[code] != len(remote): self.push(code) break except Exception as e: # noqa: BLE001 if attempt == attempts - 1: raise RuntimeError(f"could not read the answers from the private repo: {e}") from e print(f"[answer-sync] restore failed ({e}); retrying", flush=True) time.sleep(self.backoff * 2 ** attempt) for p in self.out_dir.glob("*.jsonl"): # local-only files: pushed only from persistent storage if p.stem not in counts: counts[p.stem] = len(C.read_records(self.out_dir, p.stem)) if self.push_local_only: self.push(p.stem) else: print(f"[answer-sync] NOT pushing local-only {p.name}: the output dir was not set explicitly", flush=True) return counts # ------------------------------------------------------------------ pushing def push(self, code: str) -> None: """Queue this annotator's file for upload. Several pushes of one code before it is uploaded coalesce.""" with self._lock: if code in self._pending: return self._pending.add(code) self._q.put(code) def _requeue_failed(self) -> None: for code in list(self.failed): self.push(code) def _run(self) -> None: while True: try: first = self._q.get(timeout=60) except queue.Empty: # idle: retry anything that failed earlier self._requeue_failed() continue codes = [first] try: # One commit per interval, carrying every file queued meanwhile: the Hub has an hourly commit quota # (not documented; "exceeded our hourly quotas for action: commit"), and one commit per finished # document reaches it when several people work at once. flush() cuts the wait short. self._wake.wait(max(0.0, self._last_commit + self.min_interval - time.monotonic())) while True: try: codes.append(self._q.get_nowait()) except queue.Empty: break with self._lock: self._pending.difference_update(codes) # a write after this point re-queues the code self._push_batch(sorted(set(codes) | self.failed)) # earlier failures ride along finally: for _ in codes: self._q.task_done() def _push_batch(self, codes: list[str]) -> None: for attempt in range(self.retries): try: files = {f"{self.prefix}/{c}.jsonl": self._merge_into_local(c, self._remote(c)) # never shrink for c in codes} files = {k: v for k, v in files.items() if v} if files: self.store.upload_many(files, f"answers: {', '.join(codes)}") self._last_commit = time.monotonic() self.pushed += len(files) self.failed.difference_update(codes) return except Exception as e: # noqa: BLE001 -- network, rate limit, hub outage: retry, never crash print(f"[answer-sync] push of {', '.join(codes)} failed (attempt {attempt + 1}/{self.retries}): {e}", flush=True) time.sleep(self.backoff * 2 ** attempt) self._last_commit = time.monotonic() self.failed.update(codes) print(f"[answer-sync] giving up on {', '.join(codes)} for now; safe on local disk, retried with the next push " f"or within a minute", flush=True) def flush(self, timeout: float = 30.0) -> bool: """Wait until every queued push is done (or timeout). Also retries codes that failed earlier. For shutdown: from here on pushes no longer wait for the commit interval.""" self._wake.set() self._requeue_failed() t0 = time.time() while self._q.unfinished_tasks: if time.time() - t0 > timeout: return False time.sleep(0.1) return not self.failed