Upload folder using huggingface_hub

.gitignore

@@ -0,0 +1,10 @@
+# NEVER commit secrets or local response data
+Keys.txt
+keys.txt
+*.token
+.env
+local_responses/
+__pycache__/
+*.pyc
+# real patient images should come from a PRIVATE CASES_DATASET, not the repo:
+data/img/*_real*

README.md

@@ -1,13 +1,67 @@
 ---
-title: Grace Reader Study
-emoji: 🦀
-colorFrom: yellow
-colorTo: green
+title: GRACE Reader Study
+emoji: 🩺
+colorFrom: indigo
+colorTo: blue
 sdk: gradio
-sdk_version: 6.19.0
-python_version: '3.13'
+sdk_version: 4.44.1
 app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# GRACE Reader Study annotation app
+
+Blinded, resume-safe radiologist reader study for the GRACE project. One case per screen,
+no scrolling: reference CXR on the left, each anonymized system's output on the right with its
+rating controls directly beneath the image it refers to. Scale legends and subjective-term
+definitions are printed inline; per-image zoom / brightness / contrast are display-only.
+
+## What a reader does per case
+For each anonymized item: **answer correctness** (Correct / Incorrect / Indeterminate),
+**decision appropriateness** (1-5), **grounding relevance** (Relevant / Partial / Not relevant),
+optional note. Plus one case-level question: is the reference (ground-truth) annotation acceptable.
+No best/worst ranking is asked; rankings are derived in the backend from per-item scores.
+
+## Run locally
+```bash
+pip install -r requirements.txt
+python build_cases_example.py      # generates data/cases.json + placeholder images (demo)
+python app.py                      # http://127.0.0.1:7860
+```
+Demo credentials (when READER_CREDENTIALS is unset): `reader1 / changeme`, `reader2 / changeme`.
+
+## Deploy as a Hugging Face Space
+1. Create a **Gradio** Space.
+2. Upload `app.py`, `requirements.txt`, `README.md`, and (for a demo) the `data/` folder.
+   For the real study do NOT commit patient images: put `cases.json` + images in a **private
+   dataset** and set `CASES_DATASET` instead (see below).
+3. In **Settings -> Variables and secrets**, add these **secrets** (never commit them):
+   - `HF_TOKEN` = your HF token with **write** permission (the value in `Keys.txt`; do not paste it into any file).
+   - `READER_CREDENTIALS` = JSON, e.g. `{"Dr Kotak":"<pw1>","Dr B":"<pw2>","Resident C":"<pw3>"}`.
+   - `RESPONSE_DATASET` = e.g. `DrSyedFaizan/grace-reader-responses` (created automatically, private).
+   - `CASES_DATASET` (optional) = e.g. `DrSyedFaizan/grace-reader-cases` (private; pulled at boot).
+   - `APP_SECRET` (optional) = any random string that signs resume tokens.
+
+## Data flow and storage
+- Responses stream to the private dataset `RESPONSE_DATASET` on every **Save & Next**, as
+  `responses/<reader>.jsonl`, and to a local backup under `local_responses/` (so an HF write
+  hiccup never loses data).
+- **Robust schema:** one append-only record per `(annotator, case_id, item_id)` with a
+  `dims` dict of value-per-dimension, plus a `__case__` record for case-level answers. Later
+  changes to layout or wording can never overwrite or invalidate saved annotations; reads take
+  the latest record per key.
+
+## Auth / session / resume
+- First visit always lands on the login page; the app is gated by per-reader credentials.
+- On login the reader resumes at their first unfinished case and sees a live `X / N` counter
+  (their own completed count, not a reset to 1). Item order is shuffled deterministically per
+  `(reader, case)` so a resumed case looks identical.
+- Closing the tab does NOT log out (a signed token is kept in `localStorage`); use the explicit
+  **Logout** button to end the session.
+
+## Preparing the real cases.json
+Run `build_cases_example.py` and read its docstring for the exact schema. Each item's `item_id`
+is the TRUE system id (used only in the backend for the blinded GRACE-vs-baseline comparison and
+derived rankings); the reader never sees it. Keep `items` per case small (2-3) so the no-scroll
+goal holds. Enrich the case set per `../reader_protocol.md` (near-boundary, deferrals, errors,
+rare pathology, easy anchors).

app.py

@@ -0,0 +1,568 @@
+"""
+GRACE Reader Study - expert radiologist annotation app (Gradio / Hugging Face Space).
+
+Design goal: annotating one case requires ZERO scrolling and ZERO guessing.
+Everything needed to judge and score a case is visible at once:
+  - two-column layout, one row per rated item (left = reference CXR, right = the item to score);
+  - every rating control sits directly below the image it refers to;
+  - full scale legends printed inline, subjective definitions in bold red with a worked example;
+  - per-image zoom / brightness / contrast controls (display-only, never touch stored data);
+  - no best/worst ranking asked; rankings are DERIVED in the backend from per-item scores.
+
+Secrets/config come from environment variables (set them as HF Space secrets):
+  HF_TOKEN            : HF token with write permission (for the private response dataset).
+  RESPONSE_DATASET    : private dataset repo id for responses (default DrSyedFaizan/grace-reader-responses).
+  CASES_DATASET       : optional private dataset holding cases.json + images (snapshot_download at boot).
+  READER_CREDENTIALS  : JSON string {"reader_name": "password", ...}. If unset, a demo login is used (warned).
+  APP_SECRET          : secret string used to sign resume tokens (defaults derived from HF_TOKEN).
+
+NEVER commit Keys.txt or any token into the Space repo (see .gitignore).
+"""
+
+import os
+import io
+import json
+import time
+import base64
+import random
+import hashlib
+import hmac
+from pathlib import Path
+
+import gradio as gr
+from PIL import Image
+
+# ----------------------------------------------------------------------------- config
+APP_DIR = Path(__file__).parent
+DATA_DIR = APP_DIR / "data"
+LOCAL_BACKUP_DIR = APP_DIR / "local_responses"
+LOCAL_BACKUP_DIR.mkdir(exist_ok=True)
+
+HF_TOKEN = os.environ.get("HF_TOKEN", "").strip()
+RESPONSE_DATASET = os.environ.get("RESPONSE_DATASET", "DrSyedFaizan/grace-reader-responses").strip()
+CASES_DATASET = os.environ.get("CASES_DATASET", "").strip()
+APP_SECRET = os.environ.get("APP_SECRET", "").strip() or ("grace-" + hashlib.sha256(HF_TOKEN.encode()).hexdigest()[:16] if HF_TOKEN else "grace-dev-secret")
+
+MAX_ITEMS = 6            # max anonymized items rated per case; keep cases small (2-3) to preserve no-scroll
+DISPLAY_MAX_W = 820      # px, display-copy width cap (does not affect stored data)
+SCHEMA_VERSION = 1
+
+try:
+    _creds = json.loads(os.environ.get("READER_CREDENTIALS", "").strip() or "{}")
+except Exception:
+    _creds = {}
+if not _creds:
+    _creds = {"reader1": "changeme", "reader2": "changeme"}
+    print("[WARN] READER_CREDENTIALS not set - using demo credentials. Set the secret before the real study.")
+READERS = {str(k): str(v) for k, v in _creds.items()}
+
+# ----------------------------------------------------------------------------- HF api
+try:
+    from huggingface_hub import HfApi
+    _api = HfApi(token=HF_TOKEN) if HF_TOKEN else None
+except Exception as e:      # library missing at import time
+    _api = None
+    print(f"[WARN] huggingface_hub unavailable: {e}")
+
+
+def _ensure_response_dataset():
+    if not _api:
+        return
+    try:
+        _api.create_repo(RESPONSE_DATASET, repo_type="dataset", private=True, exist_ok=True)
+    except Exception as e:
+        print(f"[WARN] could not ensure response dataset: {e}")
+
+
+# ----------------------------------------------------------------------------- cases
+def _maybe_pull_cases():
+    """If CASES_DATASET is set, snapshot it into ./data (cases.json + images)."""
+    if not CASES_DATASET or not _api:
+        return
+    try:
+        from huggingface_hub import snapshot_download
+        snapshot_download(CASES_DATASET, repo_type="dataset", local_dir=str(DATA_DIR),
+                          token=HF_TOKEN, local_dir_use_symlinks=False)
+        print(f"[info] pulled cases from {CASES_DATASET}")
+    except Exception as e:
+        print(f"[WARN] could not pull CASES_DATASET: {e}")
+
+
+def load_cases():
+    _maybe_pull_cases()
+    cj = DATA_DIR / "cases.json"
+    if not cj.exists():
+        print("[WARN] no data/cases.json found - run build_cases_example.py to generate a demo set.")
+        return []
+    with open(cj, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    return data.get("cases", [])
+
+
+CASES = load_cases()
+N_CASES = len(CASES)
+CASE_BY_ID = {c["case_id"]: c for c in CASES}
+
+_IMG_CACHE = {}
+
+
+def img_data_uri(rel_path):
+    """Load an image, resize a display copy, return a base64 data URI (cached)."""
+    if not rel_path:
+        return ""
+    if rel_path in _IMG_CACHE:
+        return _IMG_CACHE[rel_path]
+    p = DATA_DIR / rel_path
+    try:
+        im = Image.open(p).convert("RGB")
+        if im.width > DISPLAY_MAX_W:
+            h = int(im.height * DISPLAY_MAX_W / im.width)
+            im = im.resize((DISPLAY_MAX_W, h))
+        buf = io.BytesIO()
+        im.save(buf, format="PNG")
+        uri = "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
+    except Exception as e:
+        print(f"[WARN] image load failed {p}: {e}")
+        uri = ""
+    _IMG_CACHE[rel_path] = uri
+    return uri
+
+
+# ----------------------------------------------------------------------------- tokens/auth
+def make_token(name):
+    return hmac.new(APP_SECRET.encode(), name.encode(), hashlib.sha256).hexdigest()
+
+
+def valid_token(name, tok):
+    return bool(name) and bool(tok) and hmac.compare_digest(make_token(name), tok)
+
+
+# ----------------------------------------------------------------------------- storage
+def _resp_path(annotator):
+    return f"responses/{annotator}.jsonl"
+
+
+def load_existing_responses(annotator):
+    """Return list of records for this annotator (from HF dataset, else local backup)."""
+    records = []
+    if _api:
+        try:
+            from huggingface_hub import hf_hub_download
+            fp = hf_hub_download(RESPONSE_DATASET, _resp_path(annotator), repo_type="dataset", token=HF_TOKEN)
+            with open(fp, "r", encoding="utf-8") as f:
+                records = [json.loads(l) for l in f if l.strip()]
+            return records
+        except Exception:
+            pass
+    lb = LOCAL_BACKUP_DIR / f"{annotator}.jsonl"
+    if lb.exists():
+        with open(lb, "r", encoding="utf-8") as f:
+            records = [json.loads(l) for l in f if l.strip()]
+    return records
+
+
+def completed_case_ids(records):
+    return {r["case_id"] for r in records if r.get("item_id") == "__case__"}
+
+
+def save_records(annotator, new_records):
+    """Append records (append-only, keyed by (annotator, case_id, item_id, dimension))
+    and stream the full per-annotator file to the private dataset + a local backup."""
+    existing = load_existing_responses(annotator)
+    existing.extend(new_records)
+    payload = "\n".join(json.dumps(r, ensure_ascii=False) for r in existing) + "\n"
+
+    # local backup first (never lose data even if HF write fails)
+    with open(LOCAL_BACKUP_DIR / f"{annotator}.jsonl", "w", encoding="utf-8") as f:
+        f.write(payload)
+
+    if _api:
+        try:
+            _api.upload_file(
+                path_or_fileobj=payload.encode("utf-8"),
+                path_in_repo=_resp_path(annotator),
+                repo_id=RESPONSE_DATASET,
+                repo_type="dataset",
+                commit_message=f"responses {annotator} {int(time.time())}",
+            )
+        except Exception as e:
+            print(f"[WARN] HF upload failed (kept local backup): {e}")
+    return existing
+
+
+# ----------------------------------------------------------------------------- ordering
+def item_order(annotator, case):
+    """Deterministic per-(annotator, case) shuffle so resume shows the same order."""
+    items = list(case.get("items", []))
+    seed = int(hashlib.sha256(f"{annotator}|{case['case_id']}".encode()).hexdigest(), 16) % (2**32)
+    rnd = random.Random(seed)
+    rnd.shuffle(items)
+    return items
+
+
+def first_unfinished(records):
+    done = completed_case_ids(records)
+    for i, c in enumerate(CASES):
+        if c["case_id"] not in done:
+            return i
+    return N_CASES     # all done
+
+
+# ----------------------------------------------------------------------------- UI text
+def image_html(dom_id, uri):
+    """One image with its own display-only zoom/brightness/contrast strip."""
+    if not uri:
+        return f"<div class='imgcell'><div class='imgbox empty'>[no image]</div></div>"
+    return f"""
+<div class='imgcell'>
+  <div class='imgbox'><img id='{dom_id}' src='{uri}' data-bright='1' data-contrast='1' data-zoom='1'></div>
+  <div class='strip'>
+    <button type='button' onclick="graceZoom('{dom_id}',-0.25)" title='zoom out'>&minus;</button>
+    <span class='lbl'>zoom</span>
+    <button type='button' onclick="graceZoom('{dom_id}',0.25)" title='zoom in'>+</button>
+    <label class='lbl'>bright<input type='range' min='0.3' max='2.5' step='0.05' value='1'
+        oninput="graceSet('{dom_id}','bright',this.value)"></label>
+    <label class='lbl'>contrast<input type='range' min='0.3' max='2.5' step='0.05' value='1'
+        oninput="graceSet('{dom_id}','contrast',this.value)"></label>
+  </div>
+</div>"""
+
+
+APPROP_LEGEND = ("**Decision appropriateness (1-5):**  "
+                 "1 = clearly inappropriate,  2 = probably inappropriate,  3 = borderline / unsure,  "
+                 "4 = probably appropriate,  5 = clearly appropriate.")
+GROUND_DEF = ("<span class='reddef'><b>Grounding relevance</b> = does the highlighted region sit on the "
+              "actual finding? <b>Example:</b> a right-lower-lobe opacity with the highlight on the right "
+              "lower lobe = <b>Relevant</b>; the same case with the highlight on the heart border = "
+              "<b>Not relevant</b>.</span>")
+APPROP_DEF = ("<span class='reddef'><b>Appropriate</b> = the system's choice to answer vs defer was the safe, "
+              "correct call for this image. <b>Example:</b> a subtle, ambiguous nodule where the system "
+              "<b>defers to a radiologist</b> = appropriate; an obvious large opacity it needlessly defers = "
+              "inappropriate.</span>")
+
+INTRO_DEFAULT = (
+    "The item(s) below come from anonymized automated reading systems, shown in random order with all "
+    "identifying names hidden. For each item you see the system's <b>answer</b> to the clinical question, "
+    "whether it chose to <b>answer</b> or <b>defer to a radiologist</b>, and a <b>highlight</b> of the image "
+    "region it relied on. The final row shows the <b>reference region</b> (ground-truth annotation) for "
+    "comparison. You cannot tell which system produced which item, and that is intentional.")
+
+
+# ----------------------------------------------------------------------------- head (JS + CSS)
+HEAD = """
+<style>
+  .imgcell { display:flex; flex-direction:column; gap:4px; }
+  .imgbox { overflow:auto; max-height:270px; border:1px solid #d0d5dd; border-radius:6px; background:#0b0b0b; }
+  .imgbox.empty { display:flex; align-items:center; justify-content:center; color:#999; height:120px; background:#f3f4f6; }
+  .imgbox img { display:block; max-width:100%; }
+  .strip { display:flex; align-items:center; gap:8px; flex-wrap:wrap; font-size:12px; }
+  .strip button { width:26px; height:24px; font-weight:700; cursor:pointer; }
+  .strip .lbl { color:#475467; }
+  .strip input[type=range] { width:90px; vertical-align:middle; }
+  .reddef { color:#c0261c; font-size:13px; display:block; margin:2px 0 6px; }
+  .legend { font-size:13px; color:#344054; }
+  .refcol { border-right:2px dashed #cbd5e1; padding-right:8px; }
+  .colhead { font-weight:700; font-size:13px; color:#101828; margin-bottom:2px; }
+  .answerbox { background:#eef2ff; border:1px solid #c7d2fe; border-radius:6px; padding:6px 8px; font-size:13px; margin:4px 0; }
+</style>
+<script>
+  window.graceApplyFilter = function(id){
+    var img = document.getElementById(id); if(!img) return;
+    var b = img.dataset.bright||1, c = img.dataset.contrast||1, z = img.dataset.zoom||1;
+    img.style.filter = 'brightness('+b+') contrast('+c+')';
+    img.style.transform = 'scale('+z+')'; img.style.transformOrigin = 'top left';
+  };
+  window.graceSet = function(id, kind, val){
+    var img = document.getElementById(id); if(!img) return;
+    img.dataset[kind] = val; window.graceApplyFilter(id);
+  };
+  window.graceZoom = function(id, delta){
+    var img = document.getElementById(id); if(!img) return;
+    var z = parseFloat(img.dataset.zoom||1) + delta;
+    if(z<0.2) z=0.2; if(z>6) z=6; img.dataset.zoom=z; window.graceApplyFilter(id);
+  };
+  // best-effort hover tooltips on the rating radios (meaning is also always shown inline)
+  window.graceTooltips = function(){
+    var map = {
+      'Correct':'model answer matches the reference / your read',
+      'Incorrect':'model answer does not match',
+      'Indeterminate':'cannot tell from this image',
+      'Relevant':'highlight sits on the actual finding',
+      'Partial':'highlight partly overlaps the finding',
+      'Not relevant':'highlight is on the wrong region',
+      '1':'clearly inappropriate','2':'probably inappropriate','3':'borderline / unsure',
+      '4':'probably appropriate','5':'clearly appropriate',
+      'yes':'reference annotation is acceptable','partial':'reference partly acceptable','no':'reference not acceptable'
+    };
+    document.querySelectorAll('label').forEach(function(l){
+      var t=(l.textContent||'').trim(); if(map[t]) l.title=map[t];
+    });
+  };
+  setInterval(function(){ try{ window.graceTooltips(); }catch(e){} }, 1500);
+</script>
+"""
+
+# ----------------------------------------------------------------------------- app
+def build_app():
+    with gr.Blocks(title="GRACE Reader Study", head=HEAD, theme=gr.themes.Soft()) as demo:
+        annotator_state = gr.State("")
+        index_state = gr.State(0)
+        order_state = gr.State([])          # true item_ids in shown order for current case
+
+        # hidden helpers for localStorage resume
+        user_ls = gr.Textbox(visible=False)
+        tok_ls = gr.Textbox(visible=False)
+        tok_out = gr.Textbox(visible=False)  # token produced on login, written to localStorage
+
+        # ------------------------------------------------------------- LOGIN VIEW
+        with gr.Column(visible=True) as login_col:
+            gr.Markdown("## GRACE Reader Study - sign in")
+            gr.Markdown("Enter your reader name and password. You can stop and resume anytime; "
+                        "closing the tab does **not** sign you out.")
+            name_in = gr.Textbox(label="Reader name", placeholder="your name")
+            pass_in = gr.Textbox(label="Password", type="password")
+            login_btn = gr.Button("Sign in", variant="primary")
+            login_msg = gr.Markdown("")
+
+        # ------------------------------------------------------------- STUDY VIEW
+        with gr.Column(visible=False) as study_col:
+            with gr.Row():
+                progress_md = gr.Markdown("0 / 0")
+                logout_btn = gr.Button("Logout", scale=0)
+            intro_html = gr.HTML("")     # per-case provenance note (shown before the resume line)
+            gr.Markdown("You can stop and resume anytime. Your answers save the moment you press "
+                        "**Save & Next**.")
+            question_md = gr.Markdown("")
+
+            # header row
+            with gr.Row():
+                gr.Markdown("<div class='colhead refcol'>REFERENCE image (for comparison)</div>")
+                gr.Markdown("<div class='colhead'>ITEM to score</div>")
+
+            gr.Markdown(f"<div class='legend'>{APPROP_LEGEND}</div>")
+            gr.HTML(APPROP_DEF)
+            gr.HTML(GROUND_DEF)
+
+            item_rows, ref_htmls, item_htmls, ans_mds = [], [], [], []
+            corr_rs, appr_rs, grnd_rs, note_tbs = [], [], [], []
+            for i in range(MAX_ITEMS):
+                with gr.Group(visible=False) as row:
+                    with gr.Row():
+                        with gr.Column(scale=1):
+                            ref_htmls.append(gr.HTML("", elem_classes="refcol"))
+                        with gr.Column(scale=1):
+                            item_htmls.append(gr.HTML(""))
+                            ans_mds.append(gr.Markdown(""))
+                            corr_rs.append(gr.Radio(["Correct", "Incorrect", "Indeterminate"],
+                                                    label="Answer correctness"))
+                            appr_rs.append(gr.Radio(["1", "2", "3", "4", "5"],
+                                                    label="Decision appropriateness (1-5)"))
+                            grnd_rs.append(gr.Radio(["Relevant", "Partial", "Not relevant"],
+                                                    label="Grounding relevance"))
+                            note_tbs.append(gr.Textbox(label="Note (optional)", lines=1))
+                item_rows.append(row)
+
+            # ground-truth / reference-ROI row + case-level question
+            gr.Markdown("<div class='colhead'>Reference region (ground truth)</div>")
+            with gr.Row():
+                gt_ref_html = gr.HTML("", elem_classes="refcol")
+                gt_html = gr.HTML("")
+            case_acceptable = gr.Radio(
+                ["yes", "partial", "no"],
+                label="Is the reference (ground-truth) annotation acceptable for this case?")
+
+            status_md = gr.Markdown("")
+            save_btn = gr.Button("Save & Next", variant="primary")
+            done_md = gr.Markdown("", visible=False)
+
+        # ---- ordered output list for render (must match render_case return order).
+        # NOTE: status_md is deliberately NOT here; it is driven only by the save/login
+        # handlers (listing it twice would make it a duplicate output and clobber messages).
+        CASE_OUTPUTS = ([intro_html, question_md, gt_ref_html, gt_html, case_acceptable,
+                         progress_md, done_md]
+                        + ref_htmls + item_htmls + ans_mds
+                        + corr_rs + appr_rs + grnd_rs + note_tbs + item_rows)
+
+        def render_case(annotator, idx, records=None):
+            if records is None:
+                records = load_existing_responses(annotator)
+            done_n = len(completed_case_ids(records))
+            base = {
+                "progress": f"**{done_n} / {N_CASES}** cases completed",
+            }
+            # finished everything
+            if idx >= N_CASES:
+                ups = [gr.update(value=""), gr.update(value=""), gr.update(value=""),
+                       gr.update(value=""), gr.update(value=None),
+                       gr.update(value=base["progress"]),
+                       gr.update(value="### All cases complete. Thank you.", visible=True)]
+                ups += [gr.update(value="") for _ in ref_htmls]
+                ups += [gr.update(value="") for _ in item_htmls]
+                ups += [gr.update(value="") for _ in ans_mds]
+                ups += [gr.update(value=None) for _ in corr_rs]
+                ups += [gr.update(value=None) for _ in appr_rs]
+                ups += [gr.update(value=None) for _ in grnd_rs]
+                ups += [gr.update(value="") for _ in note_tbs]
+                ups += [gr.update(visible=False) for _ in item_rows]
+                return ups, []
+
+            case = CASES[idx]
+            ref_uri = img_data_uri(case.get("reference_image", ""))
+            items = item_order(annotator, case)
+            order_ids = [it["item_id"] for it in items]
+            intro = case.get("intro") or INTRO_DEFAULT
+            q = "### " + case.get("question", "Assess the finding in this chest X-ray.")
+
+            ref_ups, item_ups, ans_ups, corr_ups, appr_ups, grnd_ups, note_ups, row_ups = \
+                [], [], [], [], [], [], [], []
+            for i in range(MAX_ITEMS):
+                if i < len(items):
+                    it = items[i]
+                    ref_ups.append(gr.update(value=image_html(f"ref_{idx}_{i}", ref_uri)))
+                    item_ups.append(gr.update(value=image_html(f"item_{idx}_{i}", img_data_uri(it.get("image", "")))))
+                    dec = it.get("decision", "answer")
+                    ans_ups.append(gr.update(value=(f"<div class='answerbox'><b>System decision:</b> "
+                                                    f"{'ANSWERED' if dec=='answer' else 'DEFERRED to radiologist'}"
+                                                    f"<br><b>Answer:</b> {it.get('answer','(none)')}</div>")))
+                    corr_ups.append(gr.update(value=None))
+                    appr_ups.append(gr.update(value=None))
+                    grnd_ups.append(gr.update(value=None))
+                    note_ups.append(gr.update(value=""))
+                    row_ups.append(gr.update(visible=True))
+                else:
+                    ref_ups.append(gr.update(value=""))
+                    item_ups.append(gr.update(value=""))
+                    ans_ups.append(gr.update(value=""))
+                    corr_ups.append(gr.update(value=None))
+                    appr_ups.append(gr.update(value=None))
+                    grnd_ups.append(gr.update(value=None))
+                    note_ups.append(gr.update(value=""))
+                    row_ups.append(gr.update(visible=False))
+
+            head = [
+                gr.update(value=f"<p>{intro}</p>"),
+                gr.update(value=q),
+                gr.update(value=image_html(f"gtref_{idx}", ref_uri)),
+                gr.update(value=image_html(f"gt_{idx}", img_data_uri(case.get("groundtruth_image", "")))),
+                gr.update(value=None),
+                gr.update(value=base["progress"]),
+                gr.update(value="", visible=False),
+            ]
+            ups = head + ref_ups + item_ups + ans_ups + corr_ups + appr_ups + grnd_ups + note_ups + row_ups
+            return ups, order_ids
+
+        # ------------------------------------------------------------- login logic
+        def do_login(name, pw):
+            name = (name or "").strip()
+            if name not in READERS or pw != READERS[name]:
+                return ([gr.update(), gr.update(visible=True), gr.update(visible=False),
+                         "", 0, [], gr.update(value="Invalid name or password.")]
+                        + [gr.update() for _ in CASE_OUTPUTS])
+            records = load_existing_responses(name)
+            idx = first_unfinished(records)
+            ups, order_ids = render_case(name, idx, records)
+            token = make_token(name)
+            return ([gr.update(value=token),                # tok_out -> localStorage
+                     gr.update(visible=False),              # login_col hide
+                     gr.update(visible=True),               # study_col show
+                     name, idx, order_ids,
+                     gr.update(value="")]                   # login_msg clear
+                    + ups)
+
+        def do_auto_login(name, tok):
+            name = (name or "").strip()
+            if not valid_token(name, tok) or name not in READERS:
+                return ([gr.update(), gr.update(visible=True), gr.update(visible=False),
+                         "", 0, [], gr.update()]
+                        + [gr.update() for _ in CASE_OUTPUTS])
+            records = load_existing_responses(name)
+            idx = first_unfinished(records)
+            ups, order_ids = render_case(name, idx, records)
+            return ([gr.update(value=tok), gr.update(visible=False), gr.update(visible=True),
+                     name, idx, order_ids, gr.update(value="")]
+                    + ups)
+
+        LOGIN_OUTPUTS = [tok_out, login_col, study_col, annotator_state, index_state,
+                         order_state, login_msg] + CASE_OUTPUTS
+
+        login_btn.click(do_login, [name_in, pass_in], LOGIN_OUTPUTS).then(
+            None, [name_in, tok_out], None,
+            js="(u,t)=>{ if(t){ localStorage.setItem('grace_reader_user',u); localStorage.setItem('grace_reader_token',t);} }")
+
+        # ------------------------------------------------------------- save & next
+        def do_save(annotator, idx, order_ids, case_ok, *rating_vals):
+            # rating_vals = corr[0..M], appr[0..M], grnd[0..M], note[0..M]
+            M = MAX_ITEMS
+            corr = rating_vals[0:M]
+            appr = rating_vals[M:2 * M]
+            grnd = rating_vals[2 * M:3 * M]
+            note = rating_vals[3 * M:4 * M]
+
+            if idx >= N_CASES:
+                return [idx, order_ids, gr.update(value="Nothing to save.")] + [gr.update() for _ in CASE_OUTPUTS]
+
+            n_items = len(order_ids)
+            # validation: all visible items need the three required dims; case-level needed
+            missing = []
+            for i in range(n_items):
+                if not corr[i]:
+                    missing.append(f"item {i+1}: correctness")
+                if not appr[i]:
+                    missing.append(f"item {i+1}: appropriateness")
+                if not grnd[i]:
+                    missing.append(f"item {i+1}: grounding")
+            if not case_ok:
+                missing.append("reference-acceptable question")
+            if missing:
+                msg = "Please complete before saving: " + "; ".join(missing[:6]) + ("..." if len(missing) > 6 else "")
+                return [idx, order_ids, gr.update(value=msg)] + [gr.update() for _ in CASE_OUTPUTS]
+
+            case = CASES[idx]
+            ts = int(time.time())
+            new_records = []
+            # per-item records: keyed by (annotator, case_id, item_id), value-per-dimension
+            for i in range(n_items):
+                new_records.append({
+                    "schema_version": SCHEMA_VERSION, "annotator": annotator,
+                    "case_id": case["case_id"], "item_id": order_ids[i],
+                    "shown_position": i,
+                    "dims": {"answer_correctness": corr[i], "decision_appropriateness": appr[i],
+                             "grounding_relevance": grnd[i], "note": note[i] or ""},
+                    "ts": ts,
+                })
+            # case-level record
+            new_records.append({
+                "schema_version": SCHEMA_VERSION, "annotator": annotator,
+                "case_id": case["case_id"], "item_id": "__case__",
+                "dims": {"reference_acceptable": case_ok},
+                "shown_order": order_ids, "ts": ts,
+            })
+            records = save_records(annotator, new_records)
+            nxt = first_unfinished(records)
+            ups, new_order = render_case(annotator, nxt, records)
+            return [nxt, new_order, gr.update(value="Saved.")] + ups
+
+        SAVE_INPUTS = ([annotator_state, index_state, order_state, case_acceptable]
+                       + corr_rs + appr_rs + grnd_rs + note_tbs)
+        SAVE_OUTPUTS = [index_state, order_state, status_md] + CASE_OUTPUTS
+        save_btn.click(do_save, SAVE_INPUTS, SAVE_OUTPUTS)
+
+        # ------------------------------------------------------------- logout
+        def do_logout():
+            return (gr.update(visible=True), gr.update(visible=False), "", 0, [])
+        logout_btn.click(do_logout, None,
+                         [login_col, study_col, annotator_state, index_state, order_state]).then(
+            None, None, None,
+            js="()=>{ localStorage.removeItem('grace_reader_user'); localStorage.removeItem('grace_reader_token'); location.reload(); }")
+
+        # ------------------------------------------------------------- boot / resume
+        demo.load(None, None, [user_ls, tok_ls],
+                  js="()=>[localStorage.getItem('grace_reader_user')||'', localStorage.getItem('grace_reader_token')||'']").then(
+            do_auto_login, [user_ls, tok_ls], LOGIN_OUTPUTS)
+
+    return demo
+
+
+if __name__ == "__main__":
+    _ensure_response_dataset()
+    build_app().queue().launch()

build_cases_example.py

@@ -0,0 +1,99 @@
+"""
+Generate a synthetic demo case set (data/cases.json + placeholder images) so the app
+runs immediately with ZERO real data. Replace with real P1/P3 outputs for the study:
+put real images + a real cases.json in a PRIVATE HF dataset and set CASES_DATASET.
+
+Case schema (data-driven; the app reads exactly this):
+{
+  "study": "...",
+  "cases": [
+    {
+      "case_id": "c001",
+      "question": "Is there pneumonia (lung opacity)? If present, where?",
+      "intro": "optional per-case provenance note (else app default is used)",
+      "reference_image": "img/c001_ref.png",       # the original CXR (reference column)
+      "groundtruth_image": "img/c001_gt.png",       # reference-region / bbox panel (final row)
+      "items": [                                     # anonymized systems to score, no names
+        {"item_id": "c001_sysA", "image": "img/c001_a.png",
+         "answer": "Right lower lobe opacity, consistent with pneumonia.", "decision": "answer"},
+        {"item_id": "c001_sysB", "image": "img/c001_b.png",
+         "answer": "No acute cardiopulmonary abnormality.", "decision": "defer"}
+      ]
+    }
+  ]
+}
+NOTE: item_id must be a stable TRUE id (maps to the real system in the backend); the reader
+never sees it. The app shuffles item order per (annotator, case) and derives rankings later.
+"""
+import json
+import random
+from pathlib import Path
+from PIL import Image, ImageDraw
+
+HERE = Path(__file__).parent
+DATA = HERE / "data"
+IMG = DATA / "img"
+IMG.mkdir(parents=True, exist_ok=True)
+
+random.seed(7)
+W = H = 512
+
+
+def base_cxr(seed):
+    rnd = random.Random(seed)
+    im = Image.new("RGB", (W, H), (18, 18, 18))
+    d = ImageDraw.Draw(im, "RGBA")
+    # two faint lung fields
+    for cx in (170, 342):
+        d.ellipse([cx - 90, 120, cx + 90, 400], fill=(60, 60, 60, 255))
+    # a mediastinum
+    d.rectangle([236, 120, 276, 420], fill=(40, 40, 40, 255))
+    # a random faint "opacity"
+    ox, oy = rnd.choice([(150, 320), (330, 300), (200, 200)])
+    d.ellipse([ox - 40, oy - 30, ox + 40, oy + 30], fill=(150, 150, 150, 120))
+    return im, (ox, oy)
+
+
+def box(im, center, color, label):
+    d = ImageDraw.Draw(im, "RGBA")
+    cx, cy = center
+    d.rectangle([cx - 55, cy - 45, cx + 55, cy + 45], outline=color, width=4)
+    d.rectangle([cx - 55, cy - 45, cx + 55, cy + 45], fill=color[:3] + (40,))
+    d.text((cx - 50, cy - 62), label, fill=color)
+    return im
+
+
+cases = []
+for k in range(4):
+    cid = f"c{k+1:03d}"
+    ref, opacity = base_cxr(k)
+    ref.save(IMG / f"{cid}_ref.png")
+
+    gt = ref.copy()
+    box(gt, opacity, (60, 200, 90, 255), "reference")
+    gt.save(IMG / f"{cid}_gt.png")
+
+    # system A highlights the true region (relevant); system B highlights a wrong region
+    a = ref.copy(); box(a, opacity, (230, 70, 60, 255), "highlight")
+    a.save(IMG / f"{cid}_a.png")
+    wrong = (256, 380)
+    b = ref.copy(); box(b, wrong, (230, 70, 60, 255), "highlight")
+    b.save(IMG / f"{cid}_b.png")
+
+    cases.append({
+        "case_id": cid,
+        "question": "Is there a focal lung opacity (e.g. pneumonia)? If present, where?",
+        "reference_image": f"img/{cid}_ref.png",
+        "groundtruth_image": f"img/{cid}_gt.png",
+        "items": [
+            {"item_id": f"{cid}_sysA", "image": f"img/{cid}_a.png",
+             "answer": "Focal opacity present; likely pneumonia.", "decision": "answer"},
+            {"item_id": f"{cid}_sysB", "image": f"img/{cid}_b.png",
+             "answer": "Uncertain; recommend radiologist review.", "decision": "defer"},
+        ],
+    })
+
+with open(DATA / "cases.json", "w", encoding="utf-8") as f:
+    json.dump({"study": "GRACE reader study (demo)", "cases": cases}, f, indent=2)
+
+print(f"wrote {DATA/'cases.json'} with {len(cases)} demo cases and placeholder images.")

data/cases.json

@@ -0,0 +1,85 @@
+{
+  "study": "GRACE reader study (demo)",
+  "cases": [
+    {
+      "case_id": "c001",
+      "question": "Is there a focal lung opacity (e.g. pneumonia)? If present, where?",
+      "reference_image": "img/c001_ref.png",
+      "groundtruth_image": "img/c001_gt.png",
+      "items": [
+        {
+          "item_id": "c001_sysA",
+          "image": "img/c001_a.png",
+          "answer": "Focal opacity present; likely pneumonia.",
+          "decision": "answer"
+        },
+        {
+          "item_id": "c001_sysB",
+          "image": "img/c001_b.png",
+          "answer": "Uncertain; recommend radiologist review.",
+          "decision": "defer"
+        }
+      ]
+    },
+    {
+      "case_id": "c002",
+      "question": "Is there a focal lung opacity (e.g. pneumonia)? If present, where?",
+      "reference_image": "img/c002_ref.png",
+      "groundtruth_image": "img/c002_gt.png",
+      "items": [
+        {
+          "item_id": "c002_sysA",
+          "image": "img/c002_a.png",
+          "answer": "Focal opacity present; likely pneumonia.",
+          "decision": "answer"
+        },
+        {
+          "item_id": "c002_sysB",
+          "image": "img/c002_b.png",
+          "answer": "Uncertain; recommend radiologist review.",
+          "decision": "defer"
+        }
+      ]
+    },
+    {
+      "case_id": "c003",
+      "question": "Is there a focal lung opacity (e.g. pneumonia)? If present, where?",
+      "reference_image": "img/c003_ref.png",
+      "groundtruth_image": "img/c003_gt.png",
+      "items": [
+        {
+          "item_id": "c003_sysA",
+          "image": "img/c003_a.png",
+          "answer": "Focal opacity present; likely pneumonia.",
+          "decision": "answer"
+        },
+        {
+          "item_id": "c003_sysB",
+          "image": "img/c003_b.png",
+          "answer": "Uncertain; recommend radiologist review.",
+          "decision": "defer"
+        }
+      ]
+    },
+    {
+      "case_id": "c004",
+      "question": "Is there a focal lung opacity (e.g. pneumonia)? If present, where?",
+      "reference_image": "img/c004_ref.png",
+      "groundtruth_image": "img/c004_gt.png",
+      "items": [
+        {
+          "item_id": "c004_sysA",
+          "image": "img/c004_a.png",
+          "answer": "Focal opacity present; likely pneumonia.",
+          "decision": "answer"
+        },
+        {
+          "item_id": "c004_sysB",
+          "image": "img/c004_b.png",
+          "answer": "Uncertain; recommend radiologist review.",
+          "decision": "defer"
+        }
+      ]
+    }
+  ]
+}

requirements.txt

@@ -0,0 +1,3 @@
+gradio==4.44.1
+huggingface_hub>=0.24.0
+pillow>=10.0.0