Initial deployment of caption preference study

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+.venv/bin/python filter=lfs diff=lfs merge=lfs -text
+.venv/bin/python3 filter=lfs diff=lfs merge=lfs -text
+.venv/bin/python3.12 filter=lfs diff=lfs merge=lfs -text

.venv/bin/python

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2845674b3fae665f5f6c3f01674fefa4e495f15f5d755ee8f7f1a40575abdc8a
+size 8021824

.venv/bin/python3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2845674b3fae665f5f6c3f01674fefa4e495f15f5d755ee8f7f1a40575abdc8a
+size 8021824

.venv/bin/python3.12

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2845674b3fae665f5f6c3f01674fefa4e495f15f5d755ee8f7f1a40575abdc8a
+size 8021824

.venv/pyvenv.cfg

@@ -0,0 +1,5 @@
+home = /usr/bin
+include-system-site-packages = false
+version = 3.12.3
+executable = /usr/bin/python3.12
+command = /usr/bin/python3 -m venv /scratch/Parsa2/Project-Pete/hf_experiment/.venv

README.md

@@ -1,13 +1,17 @@
 ---
 title: Caption Preference Study
-emoji: 🐨
+emoji: 🖼️
 colorFrom: indigo
-colorTo: indigo
+colorTo: purple
 sdk: gradio
-sdk_version: 6.14.0
-python_version: '3.13'
+sdk_version: 5.49.1
 app_file: app.py
 pinned: false
+short_description: A human-vs-AI image caption preference study.
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Caption Preference Study
+
+A short experiment that shows you an image and two captions. Pick whichever caption better describes the image.
+
+Your responses are stored privately by the study organizer.

app.py

@@ -0,0 +1,440 @@
+"""Caption Preference Study — Gradio Space.
+
+Participants see an image and two captions (human vs. model) and pick a
+preference. State is persisted across Space restarts via a private HF dataset.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import os
+import random
+import threading
+import time
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import gradio as gr
+import pandas as pd
+from huggingface_hub import HfApi, hf_hub_download, snapshot_download
+from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError
+
+
+HF_USER = "pmadinei"
+IMAGES_REPO = f"{HF_USER}/caption-preference-images"
+RESULTS_REPO = f"{HF_USER}/caption-preference-results"
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+RESPONSE_TIME_CAP = 100.0
+CSV_PATH = Path(__file__).parent / "Qwen3-VL-8B-Instruct.csv"
+IMAGE_DIR = Path(os.environ.get("IMAGE_DIR", "/tmp/caption_experiment_images"))
+IMAGE_DIR.mkdir(parents=True, exist_ok=True)
+
+api = HfApi(token=HF_TOKEN)
+
+
+# ---------------------------------------------------------------------------
+# Data loading
+# ---------------------------------------------------------------------------
+
+def _clean_caption(value: Any) -> str:
+    """Display captions verbatim without outer string-delimiter quotes.
+
+    pandas already unwraps CSV double-quote delimiters, but defensively strip a
+    single layer of matching outer single/double quotes if present so all
+    captions render uniformly.
+    """
+    if value is None:
+        return ""
+    text = str(value)
+    if len(text) >= 2 and text[0] == text[-1] and text[0] in ('"', "'"):
+        text = text[1:-1]
+    return text
+
+
+print(f"[startup] Loading CSV from {CSV_PATH}")
+df = pd.read_csv(CSV_PATH)
+df["human_caption"] = df["human_caption"].map(_clean_caption)
+df["model_caption"] = df["model_caption"].map(_clean_caption)
+
+_test_mask = df["image_id"].astype(str).str.contains("test", case=False, na=False)
+TEST_DF = df[_test_mask].reset_index(drop=True)
+NONTEST_DF = df[~_test_mask].reset_index(drop=True)
+
+NONTEST_IMAGE_IDS: list = list(NONTEST_DF["image_id"].unique())
+IMAGE_ID_TO_FILENAMES: dict = {
+    img_id: list(NONTEST_DF[NONTEST_DF["image_id"] == img_id]["filename"].unique())
+    for img_id in NONTEST_IMAGE_IDS
+}
+print(
+    f"[startup] {len(df)} rows | {len(NONTEST_IMAGE_IDS)} non-test image_ids | "
+    f"{len(TEST_DF)} test rows"
+)
+
+
+# ---------------------------------------------------------------------------
+# Image download
+# ---------------------------------------------------------------------------
+
+def _ensure_images_downloaded() -> None:
+    if not HF_TOKEN:
+        print("[startup] WARNING: HF_TOKEN is not set; cannot download images.")
+        return
+    print(f"[startup] Downloading images from {IMAGES_REPO} to {IMAGE_DIR}...")
+    snapshot_download(
+        repo_id=IMAGES_REPO,
+        repo_type="dataset",
+        local_dir=str(IMAGE_DIR),
+        token=HF_TOKEN,
+        max_workers=16,
+    )
+    print("[startup] Image download complete.")
+
+
+_ensure_images_downloaded()
+
+
+# ---------------------------------------------------------------------------
+# Round-robin state (persisted to RESULTS_REPO/state.json)
+# ---------------------------------------------------------------------------
+
+_STATE_LOCK = threading.Lock()
+_STATE: dict = {"image_id_used": {}}
+
+
+def _state_key(image_id: Any) -> str:
+    return str(image_id)
+
+
+def _load_state() -> None:
+    global _STATE
+    if not HF_TOKEN:
+        return
+    try:
+        path = hf_hub_download(
+            repo_id=RESULTS_REPO,
+            repo_type="dataset",
+            filename="state.json",
+            token=HF_TOKEN,
+            force_download=True,
+        )
+        with open(path) as f:
+            loaded = json.load(f)
+        _STATE = {"image_id_used": loaded.get("image_id_used", {})}
+        print(
+            f"[state] Loaded round-robin state with "
+            f"{len(_STATE['image_id_used'])} image_ids tracked."
+        )
+    except (EntryNotFoundError, RepositoryNotFoundError, FileNotFoundError):
+        print("[state] No existing state.json found, starting fresh.")
+        _STATE = {"image_id_used": {}}
+    except Exception as exc:  # noqa: BLE001
+        print(f"[state] Could not load state.json ({exc}); starting fresh.")
+        _STATE = {"image_id_used": {}}
+
+
+def _save_state() -> None:
+    if not HF_TOKEN:
+        return
+    payload = json.dumps(_STATE, indent=2).encode()
+    api.upload_file(
+        path_or_fileobj=io.BytesIO(payload),
+        path_in_repo="state.json",
+        repo_id=RESULTS_REPO,
+        repo_type="dataset",
+        commit_message="Update round-robin state",
+    )
+
+
+_load_state()
+
+
+def _assign_filenames_for_participant() -> dict:
+    """Pick one filename per non-test image_id using round-robin across participants.
+
+    For each image_id, keep a list of filenames that have been used since the last
+    reset. Choose uniformly at random from filenames NOT yet used. When all
+    filenames for an image_id have been used, reset and start a fresh cycle.
+    """
+    with _STATE_LOCK:
+        assignments: dict = {}
+        for img_id in NONTEST_IMAGE_IDS:
+            all_fns = IMAGE_ID_TO_FILENAMES[img_id]
+            key = _state_key(img_id)
+            used = list(_STATE["image_id_used"].get(key, []))
+            available = [fn for fn in all_fns if fn not in used]
+            if not available:
+                used = []
+                available = list(all_fns)
+            chosen = random.choice(available)
+            used.append(chosen)
+            _STATE["image_id_used"][key] = used
+            assignments[img_id] = chosen
+        try:
+            _save_state()
+        except Exception as exc:  # noqa: BLE001
+            print(f"[state] WARNING: could not persist state.json ({exc}).")
+    return assignments
+
+
+# ---------------------------------------------------------------------------
+# Trial construction
+# ---------------------------------------------------------------------------
+
+def _build_trials_for_participant() -> list[dict]:
+    assignments = _assign_filenames_for_participant()
+    trials: list[dict] = []
+
+    for img_id in NONTEST_IMAGE_IDS:
+        fn = assignments[img_id]
+        match = NONTEST_DF[
+            (NONTEST_DF["image_id"] == img_id) & (NONTEST_DF["filename"] == fn)
+        ]
+        if match.empty:
+            continue
+        row = match.iloc[0]
+        trials.append(_row_to_trial(row))
+
+    for _, row in TEST_DF.iterrows():
+        trials.append(_row_to_trial(row))
+
+    random.shuffle(trials)
+    return trials
+
+
+def _row_to_trial(row: pd.Series) -> dict:
+    return {
+        "id": int(row["id"]),
+        "image_id": (
+            int(row["image_id"]) if str(row["image_id"]).lstrip("-").isdigit()
+            else str(row["image_id"])
+        ),
+        "filename": str(row["filename"]),
+        "type": str(row["type"]),
+        "human_caption": str(row["human_caption"]),
+        "model_caption": str(row["model_caption"]),
+        "human_on_left": random.choice([True, False]),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Results persistence
+# ---------------------------------------------------------------------------
+
+def _save_results(session_id: str, results: list[dict]) -> None:
+    if not HF_TOKEN or not results:
+        return
+    frame = pd.DataFrame(
+        results,
+        columns=[
+            "id",
+            "image_id",
+            "filename",
+            "type",
+            "human_caption",
+            "model_caption",
+            "preference",
+            "response_time",
+        ],
+    )
+    buf = io.BytesIO()
+    frame.to_csv(buf, index=False)
+    buf.seek(0)
+    api.upload_file(
+        path_or_fileobj=buf,
+        path_in_repo=f"results/{session_id}.csv",
+        repo_id=RESULTS_REPO,
+        repo_type="dataset",
+        commit_message=f"Update results for {session_id} (n={len(results)})",
+    )
+
+
+# ---------------------------------------------------------------------------
+# Gradio handlers
+# ---------------------------------------------------------------------------
+
+WELCOME_HTML = """
+<div style="text-align:center; padding: 16px;">
+  <h2 style="margin-bottom: 8px;">Caption Preference Study</h2>
+  <p style="font-size: 1.05em;">
+    You will see images with two captions. Click the caption that better
+    describes the image.
+  </p>
+</div>
+"""
+
+DONE_HTML = """
+<div style="text-align:center; padding: 32px;">
+  <h2>All done — thank you for participating!</h2>
+  <p>You can close this tab now.</p>
+</div>
+"""
+
+
+def start_session():
+    session_id = str(uuid.uuid4())
+    trials = _build_trials_for_participant()
+    if not trials:
+        return (
+            None,
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(value="<h3>No trials available.</h3>", visible=True),
+            None,
+            "",
+            "",
+            "",
+        )
+    state = {
+        "session_id": session_id,
+        "trials": trials,
+        "current_idx": 0,
+        "trial_start_time": time.time(),
+        "results": [],
+    }
+    img_path, left, right, progress = _current_display(state)
+    return (
+        state,
+        gr.update(visible=False),
+        gr.update(visible=True),
+        gr.update(visible=False),
+        img_path,
+        left,
+        right,
+        progress,
+    )
+
+
+def _current_display(state: dict) -> tuple:
+    if state is None or state["current_idx"] >= len(state["trials"]):
+        return None, "", "", ""
+    trial = state["trials"][state["current_idx"]]
+    img_path = str(IMAGE_DIR / trial["filename"])
+    if trial["human_on_left"]:
+        left, right = trial["human_caption"], trial["model_caption"]
+    else:
+        left, right = trial["model_caption"], trial["human_caption"]
+    progress = f"Trial {state['current_idx'] + 1} of {len(state['trials'])}"
+    return img_path, left, right, progress
+
+
+def _make_choice(state: dict, side: str):
+    if state is None:
+        return state, gr.update(), gr.update(), None, "", "", ""
+    elapsed = min(time.time() - state["trial_start_time"], RESPONSE_TIME_CAP)
+    trial = state["trials"][state["current_idx"]]
+    chose_human = trial["human_on_left"] if side == "left" else not trial["human_on_left"]
+    state["results"].append(
+        {
+            "id": trial["id"],
+            "image_id": trial["image_id"],
+            "filename": trial["filename"],
+            "type": trial["type"],
+            "human_caption": trial["human_caption"],
+            "model_caption": trial["model_caption"],
+            "preference": "H" if chose_human else "M",
+            "response_time": round(elapsed, 3),
+        }
+    )
+
+    # Persist after each trial. Fire-and-forget on a background thread so the UI
+    # advances immediately; failures are logged but don't block the participant.
+    threading.Thread(
+        target=_save_results,
+        args=(state["session_id"], list(state["results"])),
+        daemon=True,
+    ).start()
+
+    state["current_idx"] += 1
+    if state["current_idx"] >= len(state["trials"]):
+        return (
+            state,
+            gr.update(visible=False),
+            gr.update(visible=True, value=DONE_HTML),
+            None,
+            "",
+            "",
+            f"Done — {len(state['trials'])} / {len(state['trials'])}",
+        )
+
+    state["trial_start_time"] = time.time()
+    img_path, left, right, progress = _current_display(state)
+    return (
+        state,
+        gr.update(visible=True),
+        gr.update(visible=False),
+        img_path,
+        left,
+        right,
+        progress,
+    )
+
+
+# ---------------------------------------------------------------------------
+# UI
+# ---------------------------------------------------------------------------
+
+custom_css = """
+.caption-btn {
+    min-height: 140px !important;
+    font-size: 1.05em !important;
+    white-space: normal !important;
+    line-height: 1.4 !important;
+    padding: 16px !important;
+    text-align: left !important;
+}
+.center-img img { max-height: 60vh !important; object-fit: contain !important; }
+"""
+
+with gr.Blocks(title="Caption Preference Study", css=custom_css) as demo:
+    state = gr.State()
+
+    intro = gr.Group(visible=True)
+    with intro:
+        gr.HTML(WELCOME_HTML)
+        with gr.Row():
+            with gr.Column(scale=1):
+                pass
+            with gr.Column(scale=1):
+                start_btn = gr.Button("Start", variant="primary", size="lg")
+            with gr.Column(scale=1):
+                pass
+
+    trial_group = gr.Group(visible=False)
+    with trial_group:
+        progress = gr.Markdown("")
+        image = gr.Image(
+            label=None,
+            show_label=False,
+            interactive=False,
+            elem_classes=["center-img"],
+        )
+        with gr.Row():
+            left_btn = gr.Button("", elem_classes=["caption-btn"])
+            right_btn = gr.Button("", elem_classes=["caption-btn"])
+
+    done_panel = gr.HTML(visible=False)
+
+    start_btn.click(
+        start_session,
+        inputs=[],
+        outputs=[state, intro, trial_group, done_panel, image, left_btn, right_btn, progress],
+    )
+
+    left_btn.click(
+        lambda s: _make_choice(s, "left"),
+        inputs=[state],
+        outputs=[state, trial_group, done_panel, image, left_btn, right_btn, progress],
+    )
+    right_btn.click(
+        lambda s: _make_choice(s, "right"),
+        inputs=[state],
+        outputs=[state, trial_group, done_panel, image, left_btn, right_btn, progress],
+    )
+
+
+if __name__ == "__main__":
+    demo.queue(default_concurrency_limit=8).launch()

requirements.txt

@@ -0,0 +1,4 @@
+gradio==5.49.1
+pandas==2.2.3
+huggingface_hub==0.35.3
+Pillow==11.0.0