File size: 15,571 Bytes
8bbbb56
 
 
 
 
 
 
 
 
8be80d9
 
8bbbb56
 
8be80d9
 
 
8bbbb56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8be80d9
 
 
 
8bbbb56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8be80d9
 
 
 
 
 
 
8bbbb56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8be80d9
8bbbb56
 
 
8be80d9
 
 
8bbbb56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8be80d9
 
8bbbb56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8be80d9
8bbbb56
 
 
8be80d9
8bbbb56
8be80d9
 
 
 
 
8bbbb56
8be80d9
 
8bbbb56
8be80d9
 
 
8bbbb56
8be80d9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8bbbb56
 
8be80d9
 
 
8bbbb56
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
"""Keep the study's private files in a PRIVATE Hugging Face dataset repo, for a public Space.

A public Space's files are readable by anyone, so the annotator codes (config) and the documents (which name the
system behind every text) cannot live there; and a Space's disk is wiped on every restart, rebuild or sleep, so
answers cannot live only there either. With ``ANNOTATION_HUB_REPO`` set, the app:

1. downloads the config and the documents from that repo at startup (``ANNOTATION_CONFIG``, ``ANNOTATION_DATA``
   are then paths inside the repo);
2. restores the existing answer files from it before serving, so resume works after any restart;
3. after every completed document, pushes that annotator's whole answer file back to it, from a background thread;
   files queued within a minute go up together in one commit (the Hub limits commits per hour).

The local write (append + fsync, core.append_record) still happens first and is what the annotator waits for. The
push follows within a minute; failures are retried with backoff and re-queued, and pending pushes are flushed on
shutdown without waiting for the minute. Repo layout: ``config.yaml``, ``data/<name>.json``,
``annotations/<data stem>/<code>.jsonl``.

The ``Store`` interface is small so tests can swap in ``LocalStore`` (a directory standing in for the repo).
"""

from __future__ import annotations

import fcntl
import os
import queue
import shutil
import tempfile
import threading
import time
from pathlib import Path

import core as C

REPO_TYPE = "dataset"


class LocalStore:
    """A directory standing in for the private repo (tests, and dry runs without network)."""

    def __init__(self, root: str | Path):
        self.root = Path(root)

    def download(self, path_in_repo: str, dest: Path) -> Path | None:
        src = self.root / path_in_repo
        if not src.exists():
            return None
        dest.parent.mkdir(parents=True, exist_ok=True)
        shutil.copyfile(src, dest)
        return dest

    def list(self, prefix: str) -> list[str]:
        base = self.root / prefix
        return sorted(str(p.relative_to(self.root)) for p in base.rglob("*") if p.is_file()) if base.exists() else []

    def upload(self, data: bytes, path_in_repo: str, message: str) -> None:
        dest = self.root / path_in_repo
        dest.parent.mkdir(parents=True, exist_ok=True)
        tmp = dest.with_suffix(dest.suffix + ".tmp")
        tmp.write_bytes(data)
        os.replace(tmp, dest)

    def upload_many(self, files: dict[str, bytes], message: str) -> None:
        for path_in_repo, data in files.items():
            self.upload(data, path_in_repo, message)


class HubStore:
    """The private dataset repo on the Hugging Face Hub."""

    def __init__(self, repo_id: str, token: str | None = None):
        from huggingface_hub import HfApi

        self.repo_id = repo_id
        self.token = token or os.environ.get("HF_TOKEN") or None
        if not self.token and os.environ.get("SPACE_ID"):
            raise RuntimeError("HF_TOKEN is not set: add a fine-grained token with read+write access to "
                               f"{repo_id} as the Space secret HF_TOKEN (make_space.py set-token), then restart")
        self.api = HfApi(token=self.token)
        info = self.api.repo_info(repo_id, repo_type=REPO_TYPE)  # fails loudly now (missing repo, bad token)
        if not getattr(info, "private", True):
            raise RuntimeError(f"{repo_id} is PUBLIC: it would expose the annotator codes and the system names. "
                               "Make it private first.")

    def download(self, path_in_repo: str, dest: Path) -> Path | None:
        """None ONLY when the file does not exist in the repo. Any other failure (network, 5xx, 429) raises:
        huggingface_hub reports those as LocalEntryNotFoundError, a subclass of EntryNotFoundError, and treating
        them as 'no file' once let a transient outage pass for 'no answers yet'."""
        from huggingface_hub import hf_hub_download
        from huggingface_hub.errors import EntryNotFoundError, LocalEntryNotFoundError

        tmp = tempfile.mkdtemp(prefix="annot_dl_")
        try:
            try:
                got = hf_hub_download(self.repo_id, path_in_repo, repo_type=REPO_TYPE, token=self.token,
                                      local_dir=tmp)
            except LocalEntryNotFoundError:
                raise
            except EntryNotFoundError:
                return None
            dest.parent.mkdir(parents=True, exist_ok=True)
            shutil.move(got, dest)
            return dest
        finally:
            shutil.rmtree(tmp, ignore_errors=True)  # every push downloads: do not leave a dir behind each time

    def list(self, prefix: str) -> list[str]:
        prefix = prefix.rstrip("/") + "/"
        return sorted(f for f in self.api.list_repo_files(self.repo_id, repo_type=REPO_TYPE) if f.startswith(prefix))

    def upload(self, data: bytes, path_in_repo: str, message: str) -> None:
        self.api.upload_file(path_or_fileobj=data, path_in_repo=path_in_repo, repo_id=self.repo_id,
                             repo_type=REPO_TYPE, commit_message=message)

    def upload_many(self, files: dict[str, bytes], message: str) -> None:
        """Several files in ONE commit."""
        from huggingface_hub import CommitOperationAdd

        self.api.create_commit(self.repo_id, repo_type=REPO_TYPE, commit_message=message,
                               operations=[CommitOperationAdd(path_in_repo=k, path_or_fileobj=v) for k, v in files.items()])


def open_store(repo: str):
    """``file:///some/dir`` = a local directory standing in for the repo (dry runs, tests); else a Hub repo id."""
    return LocalStore(repo[len("file://"):]) if repo.startswith("file://") else HubStore(repo)


def fetch(store, path_in_repo: str, cache_dir: Path) -> Path:
    """Download one required file (config, documents) into a private cache dir outside the app folder."""
    got = store.download(path_in_repo, Path(cache_dir) / path_in_repo)
    if got is None:
        raise FileNotFoundError(f"{path_in_repo} not found in the private repo")
    return got


def merge_lines(local: list[dict], remote: list[dict]) -> list[dict]:
    """Union of two answer files by doc_id. The local line wins, then remote-only lines in their order."""
    seen = {r.get("doc_id") for r in local}
    return local + [r for r in remote if r.get("doc_id") not in seen]


class AnswerSync:
    """Restores answer files at startup, re-reads an annotator's file from the repo when they log in, and pushes it
    after each completed document.

    Invariant: the repo copy only ever grows. Every push first downloads the repo copy, merges it with the local
    file (union by doc_id, under the file's exclusive lock, written back locally) and uploads the union; any error
    reading the repo copy aborts that attempt (it is retried) instead of being read as "no answers".
    """

    def __init__(self, store, out_dir: str | Path, prefix: str, push_local_only: bool = False, retries: int = 5,
                 backoff: float = 2.0, min_interval: float = 60.0):
        self.store, self.out_dir, self.prefix = store, Path(out_dir), prefix.rstrip("/")
        self.push_local_only = push_local_only  # True only for an explicit, persistent output dir
        self.retries, self.backoff = retries, backoff
        self.min_interval = min_interval   # seconds between two commits (both Spaces share one account's quota)
        self._last_commit = float("-inf")  # time.monotonic() of the last commit attempt
        self._wake = threading.Event()
        self._q: queue.Queue[str] = queue.Queue()
        self._pending: set[str] = set()
        self._lock = threading.Lock()
        self.failed: set[str] = set()      # codes whose last push failed after all retries (re-queued later)
        self.pushed = 0
        self._thread = threading.Thread(target=self._run, name="answer-sync", daemon=True)
        self._thread.start()

    # ------------------------------------------------------------------ reading the repo copy
    def _remote(self, code: str) -> list[dict]:
        """The repo's lines for this code; [] only if the repo has no file for it. Errors propagate."""
        tmp = Path(tempfile.mkdtemp(prefix="annot_pull_"))
        try:
            got = self.store.download(f"{self.prefix}/{code}.jsonl", tmp / f"{code}.jsonl")
            return C._parse_lines(got.read_text(encoding="utf-8")) if got else []
        finally:
            shutil.rmtree(tmp, ignore_errors=True)

    def _merge_into_local(self, code: str, remote: list[dict]) -> bytes:
        """Under the local file's exclusive lock: union with ``remote``, write back if it grew, return the bytes."""
        import json

        p = C.output_path(self.out_dir, code)
        if not remote and not p.exists():
            return b""  # e.g. an unknown ID typed at "I'm coming back": do not create an empty file for it
        p.parent.mkdir(parents=True, exist_ok=True)
        with open(p, "a+", encoding="utf-8") as f:
            fcntl.flock(f, fcntl.LOCK_EX)
            try:
                f.seek(0)
                local = C._parse_lines(f.read())
                merged = merge_lines(local, remote)
                data = "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in merged)
                if len(merged) != len(local):
                    f.seek(0)
                    f.truncate()
                    f.write(data)
                    f.flush()
                    os.fsync(f.fileno())
                return data.encode("utf-8")
            finally:
                fcntl.flock(f, fcntl.LOCK_UN)

    def pull(self, code: str) -> int:
        """Merge the repo's copy of this annotator's file into the local one (at login, so a second process or an
        earlier container cannot make them redo documents). Returns the number of lines; raises on Hub errors."""
        data = self._merge_into_local(code, self._remote(code))
        return sum(1 for line in data.splitlines() if line.strip())

    def remote_has(self, code: str) -> bool:
        """Whether the repo already has a file for this ID (another process may have signed it up)."""
        return f"{self.prefix}/{code}.jsonl" in self.store.list(self.prefix)

    # ------------------------------------------------------------------ startup
    def restore(self, attempts: int = 4) -> dict[str, int]:
        """Bring every remote answer file into out_dir before the app serves. A Hub error is retried, then raised:
        better a Space that fails to start (and says why) than one that serves without the answers."""
        self.out_dir.mkdir(parents=True, exist_ok=True)
        for attempt in range(attempts):
            try:
                counts = {}
                remote_codes = [Path(p).name[:-len(".jsonl")] for p in self.store.list(self.prefix)
                                if p.endswith(".jsonl")]
                for code in remote_codes:
                    remote = self._remote(code)
                    data = self._merge_into_local(code, remote)
                    counts[code] = sum(1 for line in data.splitlines() if line.strip())
                    if counts[code] != len(remote):
                        self.push(code)
                break
            except Exception as e:  # noqa: BLE001
                if attempt == attempts - 1:
                    raise RuntimeError(f"could not read the answers from the private repo: {e}") from e
                print(f"[answer-sync] restore failed ({e}); retrying", flush=True)
                time.sleep(self.backoff * 2 ** attempt)
        for p in self.out_dir.glob("*.jsonl"):             # local-only files: pushed only from persistent storage
            if p.stem not in counts:
                counts[p.stem] = len(C.read_records(self.out_dir, p.stem))
                if self.push_local_only:
                    self.push(p.stem)
                else:
                    print(f"[answer-sync] NOT pushing local-only {p.name}: the output dir was not set explicitly",
                          flush=True)
        return counts

    # ------------------------------------------------------------------ pushing
    def push(self, code: str) -> None:
        """Queue this annotator's file for upload. Several pushes of one code before it is uploaded coalesce."""
        with self._lock:
            if code in self._pending:
                return
            self._pending.add(code)
        self._q.put(code)

    def _requeue_failed(self) -> None:
        for code in list(self.failed):
            self.push(code)

    def _run(self) -> None:
        while True:
            try:
                first = self._q.get(timeout=60)
            except queue.Empty:            # idle: retry anything that failed earlier
                self._requeue_failed()
                continue
            codes = [first]
            try:
                # One commit per interval, carrying every file queued meanwhile: the Hub has an hourly commit quota
                # (not documented; "exceeded our hourly quotas for action: commit"), and one commit per finished
                # document reaches it when several people work at once. flush() cuts the wait short.
                self._wake.wait(max(0.0, self._last_commit + self.min_interval - time.monotonic()))
                while True:
                    try:
                        codes.append(self._q.get_nowait())
                    except queue.Empty:
                        break
                with self._lock:
                    self._pending.difference_update(codes)  # a write after this point re-queues the code
                self._push_batch(sorted(set(codes) | self.failed))  # earlier failures ride along
            finally:
                for _ in codes:
                    self._q.task_done()

    def _push_batch(self, codes: list[str]) -> None:
        for attempt in range(self.retries):
            try:
                files = {f"{self.prefix}/{c}.jsonl": self._merge_into_local(c, self._remote(c))  # never shrink
                         for c in codes}
                files = {k: v for k, v in files.items() if v}
                if files:
                    self.store.upload_many(files, f"answers: {', '.join(codes)}")
                self._last_commit = time.monotonic()
                self.pushed += len(files)
                self.failed.difference_update(codes)
                return
            except Exception as e:  # noqa: BLE001 -- network, rate limit, hub outage: retry, never crash
                print(f"[answer-sync] push of {', '.join(codes)} failed (attempt {attempt + 1}/{self.retries}): {e}",
                      flush=True)
                time.sleep(self.backoff * 2 ** attempt)
        self._last_commit = time.monotonic()
        self.failed.update(codes)
        print(f"[answer-sync] giving up on {', '.join(codes)} for now; safe on local disk, retried with the next push "
              f"or within a minute", flush=True)

    def flush(self, timeout: float = 30.0) -> bool:
        """Wait until every queued push is done (or timeout). Also retries codes that failed earlier. For shutdown:
        from here on pushes no longer wait for the commit interval."""
        self._wake.set()
        self._requeue_failed()
        t0 = time.time()
        while self._q.unfinished_tasks:
            if time.time() - t0 > timeout:
                return False
            time.sleep(0.1)
        return not self.failed