Spaces:
Running
Running
Download core.py from BVendeville/reading-study: direct link, hf CLI and curl.
- Browser
- Download file 22.8 kB
-
https://huggingface.co/spaces/BVendeville/reading-study/resolve/main/core.py
- Command line
-
hf download hf://spaces/BVendeville/reading-study/core.py
-
curl -L -o core.py https://huggingface.co/spaces/BVendeville/reading-study/resolve/main/core.py
22.8 kB
| """Pure logic of the annotation app (HUMAN_EVALUATION.md §4). No gradio import, so it is testable alone. | |
| Everything loaded here at startup (config, documents) is returned frozen. Per-session state is a plain dict | |
| that the UI keeps in ``gr.State``; nothing mutable lives at module level. | |
| """ | |
| from __future__ import annotations | |
| import fcntl | |
| import hashlib | |
| import json | |
| import os | |
| import random | |
| import re | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from types import MappingProxyType | |
| import yaml | |
| LETTERS = ("A", "B", "C") | |
| GAPS = ("negligible", "small", "moderate", "large") | |
| N_BLOCKS = 4 | |
| SCHEMA_VERSION = 1 | |
| EXPERT_PREFIX = "exp-" | |
| CODE_RE = re.compile(r"^[a-z]+(?:-[a-z0-9]+)+$") # prefix-..., e.g. ann-03 or ann-03-k7q2xw9m | |
| CODE_HINT = "the code you were given" | |
| ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{2,29}$") # participant-chosen IDs: 3-30 of a-z 0-9 _ - | |
| Q_EASIEST = "Which of these did you find easiest to follow?" | |
| Q_HARDEST = "Which did you find hardest to follow?" | |
| Q_OMITS = "Does the simplification leave out important information from the source?" | |
| Q_CONTRADICTS = "Does the simplification contradict the source?" | |
| Q_OVERGENERALISES = ("Does the simplification state a finding more broadly than the source supports — for example " | |
| "dropping who it applies to, or under what conditions?") | |
| def gap_question(higher: str, lower: str) -> str: | |
| return f"How much easier is {higher} than {lower}?" | |
| # ------------------------------------------------------------------------------------------------ loading | |
| def _freeze(x): | |
| if isinstance(x, dict): | |
| return MappingProxyType({k: _freeze(v) for k, v in x.items()}) | |
| if isinstance(x, list): | |
| return tuple(_freeze(v) for v in x) | |
| return x | |
| def load_data(path: str | Path) -> tuple: | |
| """Read and validate the input documents. Returns a tuple of frozen dicts, in file order.""" | |
| raw = json.loads(Path(path).read_text(encoding="utf-8")) | |
| docs = raw["documents"] if isinstance(raw, dict) else None | |
| if not isinstance(docs, list) or not docs: | |
| raise ValueError(f"{path}: expected {{\"documents\": [...]}} with at least one document") | |
| seen, systems = set(), None | |
| for i, d in enumerate(docs): | |
| for key in ("doc_id", "source", "simplifications"): | |
| if key not in d: | |
| raise ValueError(f"{path}: document {i} has no '{key}'") | |
| if d["doc_id"] in seen: | |
| raise ValueError(f"{path}: duplicate doc_id {d['doc_id']!r}") | |
| seen.add(d["doc_id"]) | |
| names = sorted(d["simplifications"]) | |
| if len(names) != len(LETTERS): | |
| raise ValueError(f"{path}: {d['doc_id']} has {len(names)} simplifications, need {len(LETTERS)}") | |
| if systems is None: | |
| systems = names | |
| elif names != systems: | |
| raise ValueError(f"{path}: {d['doc_id']} has systems {names}, the first document has {systems}") | |
| texts = [d["source"], *d["simplifications"].values()] | |
| if not all(isinstance(t, str) and t.strip() for t in texts): | |
| raise ValueError(f"{path}: {d['doc_id']} has an empty text") | |
| if len(docs) < N_BLOCKS: | |
| raise ValueError(f"{path}: {len(docs)} documents, need at least {N_BLOCKS} (one per block)") | |
| return _freeze(docs) | |
| def is_balanced_latin_square(orders: dict) -> bool: | |
| """Each block once per row and per position, and each ordered pair of blocks adjacent exactly once.""" | |
| rows = list(orders.values()) | |
| n = N_BLOCKS | |
| if len(rows) != n or any(sorted(r) != list(range(n)) for r in rows): | |
| return False | |
| if any(sorted(r[j] for r in rows) != list(range(n)) for j in range(n)): | |
| return False | |
| pairs = [(r[j], r[j + 1]) for r in rows for j in range(n - 1)] | |
| return len(pairs) == len(set(pairs)) == n * (n - 1) | |
| def is_strong_code(code: str) -> bool: | |
| """Unguessable enough for a public page: the last part has 8+ characters mixing letters and digits.""" | |
| last = code.rsplit("-", 1)[-1] | |
| return len(last) >= 8 and any(ch.isdigit() for ch in last) and any(ch.isalpha() for ch in last) | |
| def new_code(prefix: str, label: str = "") -> str: | |
| """A fresh unguessable code, e.g. new_code('exp', '03') -> 'exp-03-k7q2xw9m' (random, 10 base-36 chars).""" | |
| import secrets | |
| alphabet = "abcdefghijkmnpqrstuvwxyz23456789" # no 0/o, 1/l: codes are typed by hand | |
| while True: | |
| tail = "".join(secrets.choice(alphabet) for _ in range(10)) | |
| code = f"{prefix}-{label}-{tail}" if label else f"{prefix}-{tail}" | |
| if is_strong_code(code): | |
| return code | |
| def load_config(path: str | Path, public: bool = False, access_key: str | None = None, | |
| expert_access_key: str | None = None) -> MappingProxyType: | |
| """Keys: ``study.access_key`` is shared by every participant of the standard cohort, and the optional | |
| ``study.expert_access_key`` by every expert; the key someone signs up with decides their cohort. The arguments | |
| override the config (env ANNOTATION_ACCESS_KEY / ANNOTATION_EXPERT_ACCESS_KEY). ``public``: the app is | |
| reachable by anyone (a public Space), so a key of 6+ characters is required. | |
| ``annotators`` is optional: participants choose their own ID. IDs listed there are reserved, with a fixed | |
| order and a cohort from their prefix (``exp-`` = expert), e.g. the organiser's test ID.""" | |
| cfg = yaml.safe_load(Path(path).read_text(encoding="utf-8")) | |
| orders = {int(k): [int(b) for b in v] for k, v in cfg["orders"].items()} | |
| if not is_balanced_latin_square(orders): | |
| raise ValueError(f"{path}: 'orders' is not a balanced Latin square over blocks 0-{N_BLOCKS - 1}") | |
| annotators = {} | |
| for code, spec in (cfg.get("annotators") or {}).items(): | |
| norm = normalise_code(code) | |
| if not CODE_RE.match(norm): | |
| raise ValueError(f"{path}: annotator code {code!r} is not of the form 'prefix-...' (letters, digits, '-')") | |
| if norm in annotators: | |
| raise ValueError(f"{path}: annotator code {code!r} listed twice") | |
| if int(spec["order"]) not in orders: | |
| raise ValueError(f"{path}: annotator {code!r} has unknown order {spec['order']!r}") | |
| annotators[norm] = {"order": int(spec["order"])} | |
| study = cfg.get("study", {}) | |
| if not str(study.get("consent_markdown", "")).strip(): | |
| raise ValueError(f"{path}: study.consent_markdown is required") | |
| key = str(access_key if access_key is not None else study.get("access_key") or "").strip() | |
| expert_key = str(expert_access_key if expert_access_key is not None else study.get("expert_access_key") or "").strip() | |
| if public and len(key) < 6: | |
| raise ValueError(f"{path}: a shared access key of 6+ characters is required on a public page " | |
| f"(study.access_key in the config, or ANNOTATION_ACCESS_KEY)") | |
| if expert_key and (len(expert_key) < 6 and public or expert_key == key): | |
| raise ValueError(f"{path}: study.expert_access_key must differ from access_key (and be 6+ characters)") | |
| block_keys = {} | |
| for spec in study.get("block_keys") or []: # personal keys for a split Space: key -> its block and group | |
| k, b, c = str(spec["key"]).strip(), int(spec["block"]), spec.get("cohort", "standard") | |
| if b not in range(N_BLOCKS) or c not in ("standard", "expert") or len(k) < 6 or k in block_keys \ | |
| or k in (key, expert_key): | |
| raise ValueError(f"{path}: bad block_keys entry for block {spec.get('block')!r}: blocks 0-3, cohort " | |
| f"standard/expert, keys of 6+ characters, all different from each other and the shared keys") | |
| block_keys[k] = {"block": b, "cohort": c} | |
| study = {k: v for k, v in study.items() if k not in ("access_key", "expert_access_key", "block_keys")} | |
| return _freeze({"study": study, "orders": orders, "annotators": annotators, "access_key": key, | |
| "expert_access_key": expert_key, "block_keys": block_keys}) | |
| def check_access_key(given: str | None, config) -> bool: | |
| """True if no key is configured, or ``given`` matches one of the keys (constant-time comparison).""" | |
| return key_cohort(given, config) is not None | |
| def block_key(given: str | None, config) -> dict | None: | |
| """{"block", "cohort"} for a personal block key (constant-time comparison against each), else None.""" | |
| import hmac | |
| g = (given or "").strip().encode() | |
| hit = None | |
| for k, info in config.get("block_keys", {}).items(): | |
| if hmac.compare_digest(g, k.encode()): | |
| hit = dict(info) | |
| return hit | |
| def key_cohort(given: str | None, config) -> str | None: | |
| """The cohort a key signs people up into: "expert" for the expert key, "standard" for the shared key (or when | |
| no key is configured at all); None for a wrong key.""" | |
| import hmac | |
| g = (given or "").strip().encode() | |
| key, expert = config.get("access_key", ""), config.get("expert_access_key", "") | |
| if expert and hmac.compare_digest(g, expert.encode()): | |
| return "expert" | |
| if not key or hmac.compare_digest(g, key.encode()): | |
| return "standard" | |
| return None | |
| APP_FILES = ("app.py", "core.py", "hub.py", "guide.py") | |
| def code_hash(folder: str | Path | None = None) -> str: | |
| """sha256 (12 hex) of the app's own code files: identifies the code that produced an answer even when it is | |
| not committed.""" | |
| folder = Path(folder or Path(__file__).parent) | |
| h = hashlib.sha256() | |
| for name in APP_FILES: | |
| f = folder / name | |
| h.update(name.encode() + b"\0" + (f.read_bytes() if f.exists() else b"") + b"\0") | |
| return h.hexdigest()[:12] | |
| def app_version() -> str: | |
| """``ANNOTATION_APP_VERSION`` if set, else ``code-<hash of app.py, core.py, hub.py>``.""" | |
| return os.environ.get("ANNOTATION_APP_VERSION") or f"code-{code_hash()}" | |
| def file_sha256(path: str | Path) -> str: | |
| return hashlib.sha256(Path(path).read_bytes()).hexdigest() | |
| # ------------------------------------------------------------------------------------- codes and orders | |
| def normalise_code(code: str | None) -> str: | |
| return (code or "").strip().lower() | |
| def cohort_of(code: str) -> str: | |
| return "expert" if code.startswith(EXPERT_PREFIX) else "standard" | |
| def validate_code(code: str | None, config) -> tuple[str | None, str | None]: | |
| """Return (normalised code, None) if the code is in the config, else (None, error message).""" | |
| norm = normalise_code(code) | |
| if not norm: | |
| return None, "Please enter your annotator code." | |
| if norm not in config["annotators"]: | |
| return None, f"Unknown code “{norm}”. Check the code you were given." | |
| return norm, None | |
| def validate_id(raw: str | None) -> tuple[str | None, str | None]: | |
| """A participant-chosen ID: (normalised, None) or (None, error message).""" | |
| norm = normalise_code(raw) | |
| if not norm: | |
| return None, "Please enter an ID." | |
| if not ID_RE.match(norm): | |
| return None, "Use 3 to 30 characters: letters, digits, '-' or '_' (no spaces)." | |
| return norm, None | |
| def registration(out_dir: str | Path, code: str, config) -> dict | None: | |
| """Who this ID is: its sign-up line, or its entry in the config's reserved IDs; None if unknown.""" | |
| for r in read_records(out_dir, code): | |
| if r.get("type") == "registration": | |
| return {"cohort": r["cohort"], "order_id": r["order_id"], "block": r.get("block")} | |
| if code in config["annotators"]: | |
| return {"cohort": cohort_of(code), "order_id": config["annotators"][code]["order"], "block": None} | |
| return None | |
| def id_taken(out_dir: str | Path, code: str, config) -> bool: | |
| """Reserved, or a file with something in it (an empty file is not a sign-up: register() accepts it).""" | |
| p = output_path(out_dir, code) | |
| return code in config["annotators"] or (p.exists() and bool(p.read_text(encoding="utf-8").strip())) | |
| def assign_order(out_dir: str | Path, config, cohort: str) -> int: | |
| """Next Latin-square row for a new participant of this cohort: rows in rotation, so the four orders stay | |
| equally used within each cohort (reserved config IDs count too).""" | |
| n = sum(1 for c in config["annotators"] if cohort_of(c) == cohort) | |
| for p in Path(out_dir).glob("*.jsonl"): | |
| if p.stem not in config["annotators"]: | |
| n += any(r.get("type") == "registration" and r.get("cohort") == cohort | |
| for r in _parse_lines(p.read_text(encoding="utf-8"))) | |
| rows = sorted(config["orders"]) | |
| return rows[n % len(rows)] | |
| def register(out_dir: str | Path, code: str, cohort: str, order_id: int, block: int | None = None) -> bool: | |
| """Create this ID's file with its sign-up line. False (writes nothing) if the ID already exists.""" | |
| p = output_path(out_dir, code) | |
| p.parent.mkdir(parents=True, exist_ok=True) | |
| with open(p, "a+", encoding="utf-8") as f: | |
| fcntl.flock(f, fcntl.LOCK_EX) | |
| try: | |
| f.seek(0) | |
| if f.read().strip(): | |
| return False | |
| reg = {"type": "registration", "annotator": code, "cohort": cohort, "order_id": order_id, | |
| "registered_at": now_iso()} | |
| if block is not None: | |
| reg["block"] = block # signed up with a personal block key | |
| f.write(json.dumps(reg, ensure_ascii=False) + "\n") | |
| f.flush() | |
| os.fsync(f.fileno()) | |
| return True | |
| finally: | |
| fcntl.flock(f, fcntl.LOCK_UN) | |
| def blocks(n_docs: int) -> list[list[int]]: | |
| """Split document indices 0..n-1 into N_BLOCKS contiguous blocks; the first n % 4 blocks get one more.""" | |
| q, r = divmod(n_docs, N_BLOCKS) | |
| out, start = [], 0 | |
| for b in range(N_BLOCKS): | |
| size = q + (1 if b < r else 0) | |
| out.append(list(range(start, start + size))) | |
| start += size | |
| return out | |
| def queue_for(code: str, config, docs) -> list[str]: | |
| """A reserved (config) ID's document order.""" | |
| return queue_for_order(config["annotators"][code]["order"], config, docs) | |
| def queue_for_order(order_id: int, config, docs, split: bool = False) -> list[str]: | |
| """Document order for one Latin-square row of blocks, documents in file order within a block. | |
| ``split``: only the row's FIRST block, i.e. one quarter of the documents. The four rows start with four | |
| different blocks, so four people with orders 1-4 together cover every document exactly once.""" | |
| order = config["orders"][order_id] | |
| bl = blocks(len(docs)) | |
| return [docs[i]["doc_id"] for b in (order[:1] if split else order) for i in bl[b]] | |
| def block_of(doc_index: int, n_docs: int) -> int: | |
| return next(b for b, idx in enumerate(blocks(n_docs)) if doc_index in idx) | |
| def positions(code: str, doc_id: str, systems, salt: str = "") -> dict[str, str]: | |
| """Displayed letter -> system name. Seeded by sha256 of (salt, code, doc_id), so it is stable across processes. | |
| The salt (config ``study.position_salt``, private) stops anyone with the public code from recomputing it.""" | |
| key = f"{salt}|{code}|{doc_id}" if salt else f"{code}|{doc_id}" | |
| seed = int.from_bytes(hashlib.sha256(key.encode()).digest()[:8], "big") | |
| names = sorted(systems) | |
| random.Random(seed).shuffle(names) | |
| return dict(zip(LETTERS, names)) | |
| # ------------------------------------------------------------------------------------------- persistence | |
| def output_path(out_dir: str | Path, code: str) -> Path: | |
| return Path(out_dir) / f"{code}.jsonl" | |
| def _parse_lines(text: str) -> list[dict]: | |
| recs = [] | |
| for line in text.splitlines(): | |
| line = line.strip() | |
| if not line: | |
| continue | |
| try: | |
| recs.append(json.loads(line)) | |
| except json.JSONDecodeError: | |
| continue # a line cut by a crash mid-write: that document was not completed | |
| return recs | |
| def read_records(out_dir: str | Path, code: str) -> list[dict]: | |
| p = output_path(out_dir, code) | |
| return _parse_lines(p.read_text(encoding="utf-8")) if p.exists() else [] | |
| def done_ids(out_dir: str | Path, code: str) -> set[str]: | |
| return {r["doc_id"] for r in read_records(out_dir, code) if "doc_id" in r} | |
| def next_index(queue: list[str], done: set[str]) -> int: | |
| """Position in the queue of the first unanswered document (len(queue) when all are done).""" | |
| return next((i for i, d in enumerate(queue) if d not in done), len(queue)) | |
| def append_record(out_dir: str | Path, record: dict) -> bool: | |
| """Append one line, flushed and fsynced, under an exclusive lock. Returns False (writes nothing) if this | |
| annotator already has a line for this document, e.g. the same code open in two tabs.""" | |
| p = output_path(out_dir, record["annotator"]) | |
| p.parent.mkdir(parents=True, exist_ok=True) | |
| with open(p, "a+", encoding="utf-8") as f: | |
| fcntl.flock(f, fcntl.LOCK_EX) | |
| try: | |
| f.seek(0) | |
| if any(r.get("doc_id") == record["doc_id"] for r in _parse_lines(f.read())): | |
| return False | |
| f.seek(0, os.SEEK_END) | |
| if f.tell() and not _ends_with_newline(p): | |
| f.write("\n") # never glue a record onto a truncated line | |
| f.write(json.dumps(record, ensure_ascii=False) + "\n") | |
| f.flush() | |
| os.fsync(f.fileno()) | |
| return True | |
| finally: | |
| fcntl.flock(f, fcntl.LOCK_UN) | |
| def _ends_with_newline(p: Path) -> bool: | |
| with open(p, "rb") as g: | |
| g.seek(-1, os.SEEK_END) | |
| return g.read(1) == b"\n" | |
| def now_iso() -> str: | |
| return datetime.now(timezone.utc).isoformat(timespec="milliseconds") | |
| # ---------------------------------------------------------------------------------------------- display | |
| LIST_LINE = re.compile(r"^\s*(?:[-*•]|\d+[.)])\s") | |
| def paragraphs(text: str) -> list[list[str]]: | |
| """Display blocks of a text: every line break is a paragraph break, except that consecutive list lines stay | |
| together in one block. Sources and references separate paragraphs with a single newline, the system with a | |
| blank line; treating both the same way keeps the texts visually comparable (and not identifiable by layout).""" | |
| blocks: list[list[str]] = [] | |
| prev_list = False | |
| for line in text.replace("\r\n", "\n").split("\n"): | |
| line = line.strip() | |
| if not line: | |
| prev_list = False | |
| continue | |
| is_list = bool(LIST_LINE.match(line)) | |
| if is_list and prev_list: | |
| blocks[-1].append(line) | |
| else: | |
| blocks.append([line]) | |
| prev_list = is_list | |
| return blocks | |
| # ------------------------------------------------------------------------------------ session and screens | |
| def answers_from_other_data(out_dir: str | Path, data_sha256: str) -> dict[str, int]: | |
| """Per code, the number of saved lines answering a DIFFERENT version of the documents file (another | |
| data_file_sha256). Resume counts documents by doc_id only, so such lines would silently mark documents done.""" | |
| out = {} | |
| for p in sorted(Path(out_dir).glob("*.jsonl")): | |
| n = sum(1 for r in _parse_lines(p.read_text(encoding="utf-8")) | |
| if r.get("data_file_sha256") not in (None, data_sha256)) | |
| if n: | |
| out[p.stem] = n | |
| return out | |
| def new_session(code: str, config, docs, out_dir, split: bool = False) -> dict: | |
| """Session state for a known ID (signed up, or reserved in the config): the queue, and where to resume.""" | |
| reg = registration(out_dir, code, config) | |
| if reg is None: | |
| raise KeyError(f"unknown ID {code!r}") | |
| if split and reg.get("block") is not None: # a personal block key: exactly that block, in file order | |
| queue = [docs[i]["doc_id"] for i in blocks(len(docs))[reg["block"]]] | |
| else: | |
| queue = queue_for_order(reg["order_id"], config, docs, split) | |
| return {"code": code, "cohort": reg["cohort"], "order_id": reg["order_id"], | |
| "queue": queue, "pos": next_index(queue, done_ids(out_dir, code)), "screen": "consent", | |
| "consent_at": None, "view": None, "positions": None, "s1": None, "times": {}} | |
| def doc_by_id(docs, doc_id: str): | |
| return next(d for d in docs if d["doc_id"] == doc_id) | |
| def screen1_view(doc, pos: dict[str, str]) -> dict: | |
| """What screen 1 may contain: the three texts under their letters. Deliberately no source.""" | |
| return {"doc_id": doc["doc_id"], "texts": {L: doc["simplifications"][pos[L]] for L in LETTERS}} | |
| def screen2_view(doc, pos: dict[str, str]) -> dict: | |
| return {**screen1_view(doc, pos), "source": doc["source"]} | |
| def middle_of(easiest: str, hardest: str) -> str: | |
| return next(L for L in LETTERS if L not in (easiest, hardest)) | |
| def check_screen1(easiest, hardest, gap_bm, gap_mw) -> str | None: | |
| """Error message, or None if screen 1's answers are complete and consistent.""" | |
| if easiest not in LETTERS or hardest not in LETTERS: | |
| return "Please answer which text was easiest and which was hardest to follow." | |
| if easiest == hardest: | |
| return "The easiest and the hardest text must be different." | |
| if gap_bm not in GAPS or gap_mw not in GAPS: | |
| return "Please answer both “how much easier” questions." | |
| return None | |
| def screen1_answers(easiest, hardest, gap_bm, gap_mw) -> dict: | |
| return {"easiest": easiest, "hardest": hardest, "middle": middle_of(easiest, hardest), | |
| "gap_best_middle": gap_bm, "gap_middle_worst": gap_mw} | |
| def screen2_answers(cohort: str, omits, contradicts, overgen) -> dict: | |
| clean = lambda xs: sorted(x for x in (xs or []) if x in LETTERS) # noqa: E731 | |
| out = {"omits_important": clean(omits), "contradicts": clean(contradicts)} | |
| if cohort == "expert": | |
| out["overgeneralises"] = clean(overgen) | |
| return out | |
| def build_record(state: dict, n_docs: int, doc_index: int, s2: dict | None, meta: dict) -> dict: | |
| """One answer line. A single-task app (``meta["task"]``) leaves out the other screen's answers and times.""" | |
| task = meta.get("task", "both") | |
| rec = { | |
| "schema_version": SCHEMA_VERSION, | |
| "task": task, | |
| "annotator": state["code"], | |
| "cohort": state["cohort"], | |
| "order_id": state["order_id"], | |
| "doc_id": state["view"]["doc_id"], | |
| "doc_index": state["pos"] + 1, | |
| "n_docs": len(state["queue"]), # documents this person is asked to do | |
| "block": block_of(doc_index, n_docs), | |
| "positions": dict(state["positions"]), | |
| "screen1": state["s1"], | |
| "screen2": s2, | |
| "times": dict(state["times"]), | |
| "consent_given_at": state["consent_at"], | |
| "data_file_sha256": meta["data_sha256"], | |
| "app_version": meta["app_version"], | |
| "layout": meta.get("layout", "horizontal"), | |
| } | |
| if task == "simplicity": | |
| rec.pop("screen2") | |
| elif task == "validity": | |
| rec.pop("screen1") | |
| return rec | |