Spaces:
Sleeping
Sleeping
File size: 15,571 Bytes
8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 8be80d9 8bbbb56 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 | """Keep the study's private files in a PRIVATE Hugging Face dataset repo, for a public Space.
A public Space's files are readable by anyone, so the annotator codes (config) and the documents (which name the
system behind every text) cannot live there; and a Space's disk is wiped on every restart, rebuild or sleep, so
answers cannot live only there either. With ``ANNOTATION_HUB_REPO`` set, the app:
1. downloads the config and the documents from that repo at startup (``ANNOTATION_CONFIG``, ``ANNOTATION_DATA``
are then paths inside the repo);
2. restores the existing answer files from it before serving, so resume works after any restart;
3. after every completed document, pushes that annotator's whole answer file back to it, from a background thread;
files queued within a minute go up together in one commit (the Hub limits commits per hour).
The local write (append + fsync, core.append_record) still happens first and is what the annotator waits for. The
push follows within a minute; failures are retried with backoff and re-queued, and pending pushes are flushed on
shutdown without waiting for the minute. Repo layout: ``config.yaml``, ``data/<name>.json``,
``annotations/<data stem>/<code>.jsonl``.
The ``Store`` interface is small so tests can swap in ``LocalStore`` (a directory standing in for the repo).
"""
from __future__ import annotations
import fcntl
import os
import queue
import shutil
import tempfile
import threading
import time
from pathlib import Path
import core as C
REPO_TYPE = "dataset"
class LocalStore:
"""A directory standing in for the private repo (tests, and dry runs without network)."""
def __init__(self, root: str | Path):
self.root = Path(root)
def download(self, path_in_repo: str, dest: Path) -> Path | None:
src = self.root / path_in_repo
if not src.exists():
return None
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(src, dest)
return dest
def list(self, prefix: str) -> list[str]:
base = self.root / prefix
return sorted(str(p.relative_to(self.root)) for p in base.rglob("*") if p.is_file()) if base.exists() else []
def upload(self, data: bytes, path_in_repo: str, message: str) -> None:
dest = self.root / path_in_repo
dest.parent.mkdir(parents=True, exist_ok=True)
tmp = dest.with_suffix(dest.suffix + ".tmp")
tmp.write_bytes(data)
os.replace(tmp, dest)
def upload_many(self, files: dict[str, bytes], message: str) -> None:
for path_in_repo, data in files.items():
self.upload(data, path_in_repo, message)
class HubStore:
"""The private dataset repo on the Hugging Face Hub."""
def __init__(self, repo_id: str, token: str | None = None):
from huggingface_hub import HfApi
self.repo_id = repo_id
self.token = token or os.environ.get("HF_TOKEN") or None
if not self.token and os.environ.get("SPACE_ID"):
raise RuntimeError("HF_TOKEN is not set: add a fine-grained token with read+write access to "
f"{repo_id} as the Space secret HF_TOKEN (make_space.py set-token), then restart")
self.api = HfApi(token=self.token)
info = self.api.repo_info(repo_id, repo_type=REPO_TYPE) # fails loudly now (missing repo, bad token)
if not getattr(info, "private", True):
raise RuntimeError(f"{repo_id} is PUBLIC: it would expose the annotator codes and the system names. "
"Make it private first.")
def download(self, path_in_repo: str, dest: Path) -> Path | None:
"""None ONLY when the file does not exist in the repo. Any other failure (network, 5xx, 429) raises:
huggingface_hub reports those as LocalEntryNotFoundError, a subclass of EntryNotFoundError, and treating
them as 'no file' once let a transient outage pass for 'no answers yet'."""
from huggingface_hub import hf_hub_download
from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError
tmp = tempfile.mkdtemp(prefix="annot_dl_")
try:
try:
got = hf_hub_download(self.repo_id, path_in_repo, repo_type=REPO_TYPE, token=self.token,
local_dir=tmp)
except LocalEntryNotFoundError:
raise
except EntryNotFoundError:
return None
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.move(got, dest)
return dest
finally:
shutil.rmtree(tmp, ignore_errors=True) # every push downloads: do not leave a dir behind each time
def list(self, prefix: str) -> list[str]:
prefix = prefix.rstrip("/") + "/"
return sorted(f for f in self.api.list_repo_files(self.repo_id, repo_type=REPO_TYPE) if f.startswith(prefix))
def upload(self, data: bytes, path_in_repo: str, message: str) -> None:
self.api.upload_file(path_or_fileobj=data, path_in_repo=path_in_repo, repo_id=self.repo_id,
repo_type=REPO_TYPE, commit_message=message)
def upload_many(self, files: dict[str, bytes], message: str) -> None:
"""Several files in ONE commit."""
from huggingface_hub import CommitOperationAdd
self.api.create_commit(self.repo_id, repo_type=REPO_TYPE, commit_message=message,
operations=[CommitOperationAdd(path_in_repo=k, path_or_fileobj=v) for k, v in files.items()])
def open_store(repo: str):
"""``file:///some/dir`` = a local directory standing in for the repo (dry runs, tests); else a Hub repo id."""
return LocalStore(repo[len("file://"):]) if repo.startswith("file://") else HubStore(repo)
def fetch(store, path_in_repo: str, cache_dir: Path) -> Path:
"""Download one required file (config, documents) into a private cache dir outside the app folder."""
got = store.download(path_in_repo, Path(cache_dir) / path_in_repo)
if got is None:
raise FileNotFoundError(f"{path_in_repo} not found in the private repo")
return got
def merge_lines(local: list[dict], remote: list[dict]) -> list[dict]:
"""Union of two answer files by doc_id. The local line wins, then remote-only lines in their order."""
seen = {r.get("doc_id") for r in local}
return local + [r for r in remote if r.get("doc_id") not in seen]
class AnswerSync:
"""Restores answer files at startup, re-reads an annotator's file from the repo when they log in, and pushes it
after each completed document.
Invariant: the repo copy only ever grows. Every push first downloads the repo copy, merges it with the local
file (union by doc_id, under the file's exclusive lock, written back locally) and uploads the union; any error
reading the repo copy aborts that attempt (it is retried) instead of being read as "no answers".
"""
def __init__(self, store, out_dir: str | Path, prefix: str, push_local_only: bool = False, retries: int = 5,
backoff: float = 2.0, min_interval: float = 60.0):
self.store, self.out_dir, self.prefix = store, Path(out_dir), prefix.rstrip("/")
self.push_local_only = push_local_only # True only for an explicit, persistent output dir
self.retries, self.backoff = retries, backoff
self.min_interval = min_interval # seconds between two commits (both Spaces share one account's quota)
self._last_commit = float("-inf") # time.monotonic() of the last commit attempt
self._wake = threading.Event()
self._q: queue.Queue[str] = queue.Queue()
self._pending: set[str] = set()
self._lock = threading.Lock()
self.failed: set[str] = set() # codes whose last push failed after all retries (re-queued later)
self.pushed = 0
self._thread = threading.Thread(target=self._run, name="answer-sync", daemon=True)
self._thread.start()
# ------------------------------------------------------------------ reading the repo copy
def _remote(self, code: str) -> list[dict]:
"""The repo's lines for this code; [] only if the repo has no file for it. Errors propagate."""
tmp = Path(tempfile.mkdtemp(prefix="annot_pull_"))
try:
got = self.store.download(f"{self.prefix}/{code}.jsonl", tmp / f"{code}.jsonl")
return C._parse_lines(got.read_text(encoding="utf-8")) if got else []
finally:
shutil.rmtree(tmp, ignore_errors=True)
def _merge_into_local(self, code: str, remote: list[dict]) -> bytes:
"""Under the local file's exclusive lock: union with ``remote``, write back if it grew, return the bytes."""
import json
p = C.output_path(self.out_dir, code)
if not remote and not p.exists():
return b"" # e.g. an unknown ID typed at "I'm coming back": do not create an empty file for it
p.parent.mkdir(parents=True, exist_ok=True)
with open(p, "a+", encoding="utf-8") as f:
fcntl.flock(f, fcntl.LOCK_EX)
try:
f.seek(0)
local = C._parse_lines(f.read())
merged = merge_lines(local, remote)
data = "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in merged)
if len(merged) != len(local):
f.seek(0)
f.truncate()
f.write(data)
f.flush()
os.fsync(f.fileno())
return data.encode("utf-8")
finally:
fcntl.flock(f, fcntl.LOCK_UN)
def pull(self, code: str) -> int:
"""Merge the repo's copy of this annotator's file into the local one (at login, so a second process or an
earlier container cannot make them redo documents). Returns the number of lines; raises on Hub errors."""
data = self._merge_into_local(code, self._remote(code))
return sum(1 for line in data.splitlines() if line.strip())
def remote_has(self, code: str) -> bool:
"""Whether the repo already has a file for this ID (another process may have signed it up)."""
return f"{self.prefix}/{code}.jsonl" in self.store.list(self.prefix)
# ------------------------------------------------------------------ startup
def restore(self, attempts: int = 4) -> dict[str, int]:
"""Bring every remote answer file into out_dir before the app serves. A Hub error is retried, then raised:
better a Space that fails to start (and says why) than one that serves without the answers."""
self.out_dir.mkdir(parents=True, exist_ok=True)
for attempt in range(attempts):
try:
counts = {}
remote_codes = [Path(p).name[:-len(".jsonl")] for p in self.store.list(self.prefix)
if p.endswith(".jsonl")]
for code in remote_codes:
remote = self._remote(code)
data = self._merge_into_local(code, remote)
counts[code] = sum(1 for line in data.splitlines() if line.strip())
if counts[code] != len(remote):
self.push(code)
break
except Exception as e: # noqa: BLE001
if attempt == attempts - 1:
raise RuntimeError(f"could not read the answers from the private repo: {e}") from e
print(f"[answer-sync] restore failed ({e}); retrying", flush=True)
time.sleep(self.backoff * 2 ** attempt)
for p in self.out_dir.glob("*.jsonl"): # local-only files: pushed only from persistent storage
if p.stem not in counts:
counts[p.stem] = len(C.read_records(self.out_dir, p.stem))
if self.push_local_only:
self.push(p.stem)
else:
print(f"[answer-sync] NOT pushing local-only {p.name}: the output dir was not set explicitly",
flush=True)
return counts
# ------------------------------------------------------------------ pushing
def push(self, code: str) -> None:
"""Queue this annotator's file for upload. Several pushes of one code before it is uploaded coalesce."""
with self._lock:
if code in self._pending:
return
self._pending.add(code)
self._q.put(code)
def _requeue_failed(self) -> None:
for code in list(self.failed):
self.push(code)
def _run(self) -> None:
while True:
try:
first = self._q.get(timeout=60)
except queue.Empty: # idle: retry anything that failed earlier
self._requeue_failed()
continue
codes = [first]
try:
# One commit per interval, carrying every file queued meanwhile: the Hub has an hourly commit quota
# (not documented; "exceeded our hourly quotas for action: commit"), and one commit per finished
# document reaches it when several people work at once. flush() cuts the wait short.
self._wake.wait(max(0.0, self._last_commit + self.min_interval - time.monotonic()))
while True:
try:
codes.append(self._q.get_nowait())
except queue.Empty:
break
with self._lock:
self._pending.difference_update(codes) # a write after this point re-queues the code
self._push_batch(sorted(set(codes) | self.failed)) # earlier failures ride along
finally:
for _ in codes:
self._q.task_done()
def _push_batch(self, codes: list[str]) -> None:
for attempt in range(self.retries):
try:
files = {f"{self.prefix}/{c}.jsonl": self._merge_into_local(c, self._remote(c)) # never shrink
for c in codes}
files = {k: v for k, v in files.items() if v}
if files:
self.store.upload_many(files, f"answers: {', '.join(codes)}")
self._last_commit = time.monotonic()
self.pushed += len(files)
self.failed.difference_update(codes)
return
except Exception as e: # noqa: BLE001 -- network, rate limit, hub outage: retry, never crash
print(f"[answer-sync] push of {', '.join(codes)} failed (attempt {attempt + 1}/{self.retries}): {e}",
flush=True)
time.sleep(self.backoff * 2 ** attempt)
self._last_commit = time.monotonic()
self.failed.update(codes)
print(f"[answer-sync] giving up on {', '.join(codes)} for now; safe on local disk, retried with the next push "
f"or within a minute", flush=True)
def flush(self, timeout: float = 30.0) -> bool:
"""Wait until every queued push is done (or timeout). Also retries codes that failed earlier. For shutdown:
from here on pushes no longer wait for the commit interval."""
self._wake.set()
self._requeue_failed()
t0 = time.time()
while self._q.unfinished_tasks:
if time.time() - t0 > timeout:
return False
time.sleep(0.1)
return not self.failed
|