reading-study / hub.py
BVendeville's picture
annotation app code-24665847839a
6d5c895 verified
Raw History Blame Contribute Delete
15.6 kB
"""Keep the study's private files in a PRIVATE Hugging Face dataset repo, for a public Space.
A public Space's files are readable by anyone, so the annotator codes (config) and the documents (which name the
system behind every text) cannot live there; and a Space's disk is wiped on every restart, rebuild or sleep, so
answers cannot live only there either. With ``ANNOTATION_HUB_REPO`` set, the app:
1. downloads the config and the documents from that repo at startup (``ANNOTATION_CONFIG``, ``ANNOTATION_DATA``
are then paths inside the repo);
2. restores the existing answer files from it before serving, so resume works after any restart;
3. after every completed document, pushes that annotator's whole answer file back to it, from a background thread;
files queued within a minute go up together in one commit (the Hub limits commits per hour).
The local write (append + fsync, core.append_record) still happens first and is what the annotator waits for. The
push follows within a minute; failures are retried with backoff and re-queued, and pending pushes are flushed on
shutdown without waiting for the minute. Repo layout: ``config.yaml``, ``data/<name>.json``,
``annotations/<data stem>/<code>.jsonl``.
The ``Store`` interface is small so tests can swap in ``LocalStore`` (a directory standing in for the repo).
"""
from __future__ import annotations
import fcntl
import os
import queue
import shutil
import tempfile
import threading
import time
from pathlib import Path
import core as C
REPO_TYPE = "dataset"
class LocalStore:
"""A directory standing in for the private repo (tests, and dry runs without network)."""
def __init__(self, root: str | Path):
self.root = Path(root)
def download(self, path_in_repo: str, dest: Path) -> Path | None:
src = self.root / path_in_repo
if not src.exists():
return None
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(src, dest)
return dest
def list(self, prefix: str) -> list[str]:
base = self.root / prefix
return sorted(str(p.relative_to(self.root)) for p in base.rglob("*") if p.is_file()) if base.exists() else []
def upload(self, data: bytes, path_in_repo: str, message: str) -> None:
dest = self.root / path_in_repo
dest.parent.mkdir(parents=True, exist_ok=True)
tmp = dest.with_suffix(dest.suffix + ".tmp")
tmp.write_bytes(data)
os.replace(tmp, dest)
def upload_many(self, files: dict[str, bytes], message: str) -> None:
for path_in_repo, data in files.items():
self.upload(data, path_in_repo, message)
class HubStore:
"""The private dataset repo on the Hugging Face Hub."""
def __init__(self, repo_id: str, token: str | None = None):
from huggingface_hub import HfApi
self.repo_id = repo_id
self.token = token or os.environ.get("HF_TOKEN") or None
if not self.token and os.environ.get("SPACE_ID"):
raise RuntimeError("HF_TOKEN is not set: add a fine-grained token with read+write access to "
f"{repo_id} as the Space secret HF_TOKEN (make_space.py set-token), then restart")
self.api = HfApi(token=self.token)
info = self.api.repo_info(repo_id, repo_type=REPO_TYPE) # fails loudly now (missing repo, bad token)
if not getattr(info, "private", True):
raise RuntimeError(f"{repo_id} is PUBLIC: it would expose the annotator codes and the system names. "
"Make it private first.")
def download(self, path_in_repo: str, dest: Path) -> Path | None:
"""None ONLY when the file does not exist in the repo. Any other failure (network, 5xx, 429) raises:
huggingface_hub reports those as LocalEntryNotFoundError, a subclass of EntryNotFoundError, and treating
them as 'no file' once let a transient outage pass for 'no answers yet'."""
from huggingface_hub import hf_hub_download
from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError
tmp = tempfile.mkdtemp(prefix="annot_dl_")
try:
try:
got = hf_hub_download(self.repo_id, path_in_repo, repo_type=REPO_TYPE, token=self.token,
local_dir=tmp)
except LocalEntryNotFoundError:
raise
except EntryNotFoundError:
return None
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.move(got, dest)
return dest
finally:
shutil.rmtree(tmp, ignore_errors=True) # every push downloads: do not leave a dir behind each time
def list(self, prefix: str) -> list[str]:
prefix = prefix.rstrip("/") + "/"
return sorted(f for f in self.api.list_repo_files(self.repo_id, repo_type=REPO_TYPE) if f.startswith(prefix))
def upload(self, data: bytes, path_in_repo: str, message: str) -> None:
self.api.upload_file(path_or_fileobj=data, path_in_repo=path_in_repo, repo_id=self.repo_id,
repo_type=REPO_TYPE, commit_message=message)
def upload_many(self, files: dict[str, bytes], message: str) -> None:
"""Several files in ONE commit."""
from huggingface_hub import CommitOperationAdd
self.api.create_commit(self.repo_id, repo_type=REPO_TYPE, commit_message=message,
operations=[CommitOperationAdd(path_in_repo=k, path_or_fileobj=v) for k, v in files.items()])
def open_store(repo: str):
"""``file:///some/dir`` = a local directory standing in for the repo (dry runs, tests); else a Hub repo id."""
return LocalStore(repo[len("file://"):]) if repo.startswith("file://") else HubStore(repo)
def fetch(store, path_in_repo: str, cache_dir: Path) -> Path:
"""Download one required file (config, documents) into a private cache dir outside the app folder."""
got = store.download(path_in_repo, Path(cache_dir) / path_in_repo)
if got is None:
raise FileNotFoundError(f"{path_in_repo} not found in the private repo")
return got
def merge_lines(local: list[dict], remote: list[dict]) -> list[dict]:
"""Union of two answer files by doc_id. The local line wins, then remote-only lines in their order."""
seen = {r.get("doc_id") for r in local}
return local + [r for r in remote if r.get("doc_id") not in seen]
class AnswerSync:
"""Restores answer files at startup, re-reads an annotator's file from the repo when they log in, and pushes it
after each completed document.
Invariant: the repo copy only ever grows. Every push first downloads the repo copy, merges it with the local
file (union by doc_id, under the file's exclusive lock, written back locally) and uploads the union; any error
reading the repo copy aborts that attempt (it is retried) instead of being read as "no answers".
"""
def __init__(self, store, out_dir: str | Path, prefix: str, push_local_only: bool = False, retries: int = 5,
backoff: float = 2.0, min_interval: float = 60.0):
self.store, self.out_dir, self.prefix = store, Path(out_dir), prefix.rstrip("/")
self.push_local_only = push_local_only # True only for an explicit, persistent output dir
self.retries, self.backoff = retries, backoff
self.min_interval = min_interval # seconds between two commits (both Spaces share one account's quota)
self._last_commit = float("-inf") # time.monotonic() of the last commit attempt
self._wake = threading.Event()
self._q: queue.Queue[str] = queue.Queue()
self._pending: set[str] = set()
self._lock = threading.Lock()
self.failed: set[str] = set() # codes whose last push failed after all retries (re-queued later)
self.pushed = 0
self._thread = threading.Thread(target=self._run, name="answer-sync", daemon=True)
self._thread.start()
# ------------------------------------------------------------------ reading the repo copy
def _remote(self, code: str) -> list[dict]:
"""The repo's lines for this code; [] only if the repo has no file for it. Errors propagate."""
tmp = Path(tempfile.mkdtemp(prefix="annot_pull_"))
try:
got = self.store.download(f"{self.prefix}/{code}.jsonl", tmp / f"{code}.jsonl")
return C._parse_lines(got.read_text(encoding="utf-8")) if got else []
finally:
shutil.rmtree(tmp, ignore_errors=True)
def _merge_into_local(self, code: str, remote: list[dict]) -> bytes:
"""Under the local file's exclusive lock: union with ``remote``, write back if it grew, return the bytes."""
import json
p = C.output_path(self.out_dir, code)
if not remote and not p.exists():
return b"" # e.g. an unknown ID typed at "I'm coming back": do not create an empty file for it
p.parent.mkdir(parents=True, exist_ok=True)
with open(p, "a+", encoding="utf-8") as f:
fcntl.flock(f, fcntl.LOCK_EX)
try:
f.seek(0)
local = C._parse_lines(f.read())
merged = merge_lines(local, remote)
data = "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in merged)
if len(merged) != len(local):
f.seek(0)
f.truncate()
f.write(data)
f.flush()
os.fsync(f.fileno())
return data.encode("utf-8")
finally:
fcntl.flock(f, fcntl.LOCK_UN)
def pull(self, code: str) -> int:
"""Merge the repo's copy of this annotator's file into the local one (at login, so a second process or an
earlier container cannot make them redo documents). Returns the number of lines; raises on Hub errors."""
data = self._merge_into_local(code, self._remote(code))
return sum(1 for line in data.splitlines() if line.strip())
def remote_has(self, code: str) -> bool:
"""Whether the repo already has a file for this ID (another process may have signed it up)."""
return f"{self.prefix}/{code}.jsonl" in self.store.list(self.prefix)
# ------------------------------------------------------------------ startup
def restore(self, attempts: int = 4) -> dict[str, int]:
"""Bring every remote answer file into out_dir before the app serves. A Hub error is retried, then raised:
better a Space that fails to start (and says why) than one that serves without the answers."""
self.out_dir.mkdir(parents=True, exist_ok=True)
for attempt in range(attempts):
try:
counts = {}
remote_codes = [Path(p).name[:-len(".jsonl")] for p in self.store.list(self.prefix)
if p.endswith(".jsonl")]
for code in remote_codes:
remote = self._remote(code)
data = self._merge_into_local(code, remote)
counts[code] = sum(1 for line in data.splitlines() if line.strip())
if counts[code] != len(remote):
self.push(code)
break
except Exception as e: # noqa: BLE001
if attempt == attempts - 1:
raise RuntimeError(f"could not read the answers from the private repo: {e}") from e
print(f"[answer-sync] restore failed ({e}); retrying", flush=True)
time.sleep(self.backoff * 2 ** attempt)
for p in self.out_dir.glob("*.jsonl"): # local-only files: pushed only from persistent storage
if p.stem not in counts:
counts[p.stem] = len(C.read_records(self.out_dir, p.stem))
if self.push_local_only:
self.push(p.stem)
else:
print(f"[answer-sync] NOT pushing local-only {p.name}: the output dir was not set explicitly",
flush=True)
return counts
# ------------------------------------------------------------------ pushing
def push(self, code: str) -> None:
"""Queue this annotator's file for upload. Several pushes of one code before it is uploaded coalesce."""
with self._lock:
if code in self._pending:
return
self._pending.add(code)
self._q.put(code)
def _requeue_failed(self) -> None:
for code in list(self.failed):
self.push(code)
def _run(self) -> None:
while True:
try:
first = self._q.get(timeout=60)
except queue.Empty: # idle: retry anything that failed earlier
self._requeue_failed()
continue
codes = [first]
try:
# One commit per interval, carrying every file queued meanwhile: the Hub has an hourly commit quota
# (not documented; "exceeded our hourly quotas for action: commit"), and one commit per finished
# document reaches it when several people work at once. flush() cuts the wait short.
self._wake.wait(max(0.0, self._last_commit + self.min_interval - time.monotonic()))
while True:
try:
codes.append(self._q.get_nowait())
except queue.Empty:
break
with self._lock:
self._pending.difference_update(codes) # a write after this point re-queues the code
self._push_batch(sorted(set(codes) | self.failed)) # earlier failures ride along
finally:
for _ in codes:
self._q.task_done()
def _push_batch(self, codes: list[str]) -> None:
for attempt in range(self.retries):
try:
files = {f"{self.prefix}/{c}.jsonl": self._merge_into_local(c, self._remote(c)) # never shrink
for c in codes}
files = {k: v for k, v in files.items() if v}
if files:
self.store.upload_many(files, f"answers: {', '.join(codes)}")
self._last_commit = time.monotonic()
self.pushed += len(files)
self.failed.difference_update(codes)
return
except Exception as e: # noqa: BLE001 -- network, rate limit, hub outage: retry, never crash
print(f"[answer-sync] push of {', '.join(codes)} failed (attempt {attempt + 1}/{self.retries}): {e}",
flush=True)
time.sleep(self.backoff * 2 ** attempt)
self._last_commit = time.monotonic()
self.failed.update(codes)
print(f"[answer-sync] giving up on {', '.join(codes)} for now; safe on local disk, retried with the next push "
f"or within a minute", flush=True)
def flush(self, timeout: float = 30.0) -> bool:
"""Wait until every queued push is done (or timeout). Also retries codes that failed earlier. For shutdown:
from here on pushes no longer wait for the commit interval."""
self._wake.set()
self._requeue_failed()
t0 = time.time()
while self._q.unfinished_tasks:
if time.time() - t0 > timeout:
return False
time.sleep(0.1)
return not self.failed