conor-foley's picture
update name box
f47870b verified
Raw
History Blame Contribute Delete
23.4 kB
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""ab_test_app_dpo.py / app.py
Blinded A/B evaluation app (Macmillan vs LLM) with per-user resume + progress.
Designed for Hugging Face Spaces:
- Place `final_qna_pairs.csv` in the Space repo root.
- Set Space variables/secrets:
* HF_DATASET_REPO (public variable): e.g. conor-foley/ab-testing-logs
* HF_TOKEN (secret): a write token with access to the dataset repo
Data file:
final_qna_pairs.csv
columns: source,title,question,macmillan_answer,llm_answer
- source is an integer id (1..50)
Behaviour:
- User enters Participant ID (e.g. Ruth, Emma)
- Items shown in fixed order (1..N) based on `source`
- User can leave and return later: app reads logs/<ParticipantID>.csv from the dataset repo and resumes
- Skip counts as completed (logged as action=skip)
- A/B is randomised per item (Macmillan vs LLM)
Logging:
- Each participant gets their own CSV at logs/<ParticipantID>.csv in the dataset repo
Examples: logs/Ruth.csv, logs/Emma.csv
Local dev:
- You can run locally too. If HF_TOKEN / HF_DATASET_REPO are not set, the app will fall back to local CSV logging under results/responses_dpo.csv.
"""
from __future__ import annotations
import csv
import os
import random
import re
from datetime import datetime
from pathlib import Path
import gradio as gr
import pandas as pd
# Optional Hugging Face Hub logging (preferred on Spaces)
try:
from huggingface_hub import HfApi, hf_hub_download
from huggingface_hub.utils import EntryNotFoundError
HF_HUB_AVAILABLE = True
except Exception:
HF_HUB_AVAILABLE = False
# ----- Paths (repo-root friendly) -----
PROJECT_ROOT = Path(__file__).resolve().parent
DATA_PATH = PROJECT_ROOT / "final_qna_pairs.csv"
# Local fallback results
LOCAL_RESULTS_PATH = PROJECT_ROOT / "results" / "responses_dpo.csv"
# HF dataset repo for persistent logs (set via Space variables/secrets)
HF_DATASET_REPO = os.getenv("HF_DATASET_REPO", "").strip() # e.g. conor-foley/ab-testing-logs
HF_TOKEN = os.getenv("HF_TOKEN", "").strip()
USE_HF_DATASET = bool(HF_HUB_AVAILABLE and HF_DATASET_REPO and HF_TOKEN)
HF_API = HfApi(token=HF_TOKEN) if USE_HF_DATASET else None
# ----- Load data -----
def load_pairs(csv_path: Path) -> pd.DataFrame:
if not csv_path.exists():
raise FileNotFoundError(f"Could not find data file at {csv_path}.")
df = pd.read_csv(csv_path)
required = {"source", "title", "question", "macmillan_answer", "llm_answer"}
missing = required - set(df.columns)
if missing:
raise ValueError(f"CSV is missing required columns: {', '.join(sorted(missing))}")
# Ensure source is integer and use it as row_id
df = df.dropna(subset=["source", "question"]).copy()
df["source"] = pd.to_numeric(df["source"], errors="coerce").astype("Int64")
df = df.dropna(subset=["source"]).copy()
df["row_id"] = df["source"].astype(int)
# Sort by row_id (1..N)
df = df.sort_values("row_id").reset_index(drop=True)
return df
def clean_text(val) -> str:
s = str(val).strip()
return "" if s.lower() == "nan" else s
DF = load_pairs(DATA_PATH)
ROW_ORDER = list(DF["row_id"].tolist()) # fixed order
TOTAL_N = len(ROW_ORDER)
# Fast lookup
DF_BY_ID = {int(r.row_id): r for r in DF.itertuples(index=False)}
# ----- Logging schema -----
LOG_COLUMNS = [
"timestamp_iso",
"participant_id",
"row_id",
"title",
"action", # submit | skip
"left_item", # macmillan | llm
"right_item", # macmillan | llm
"left_text_len",
"right_text_len",
"harm_level_A",
"harm_level_B",
"harm_likelihood_A",
"harm_likelihood_B",
"pref_accurate", # A / B / About the same
"pref_relevant",
"pref_clear",
"comment",
]
def _safe_participant_id(pid: str) -> str:
"""Sanitise participant id for use in filenames.
Allows letters, numbers, underscore, dash. Strips spaces.
Example: 'Ruth' -> 'Ruth'
"""
pid = (pid or "").strip()
pid = re.sub(r"\s+", "", pid)
pid = re.sub(r"[^A-Za-z0-9_-]", "", pid)
if not pid:
raise gr.Error("Please enter a Participant ID (e.g., Ruth or Emma).")
return pid
def participant_log_filename(pid: str) -> str:
# This yields logs/Ruth.csv etc.
pid_safe = _safe_participant_id(pid)
return f"logs/{pid_safe}.csv"
# ----- Local fallback logging -----
def ensure_local_results_header(path: Path):
path.parent.mkdir(parents=True, exist_ok=True)
if not path.exists():
with path.open("w", newline="", encoding="utf-8") as f:
writer = csv.writer(f)
writer.writerow(LOG_COLUMNS)
def local_completed_row_ids(pid: str) -> set[int]:
ensure_local_results_header(LOCAL_RESULTS_PATH)
try:
df = pd.read_csv(LOCAL_RESULTS_PATH)
except Exception:
return set()
if df.empty:
return set()
sub = df[df["participant_id"].astype(str) == str(pid)]
ids = pd.to_numeric(sub["row_id"], errors="coerce").dropna().astype(int).tolist()
return set(ids)
def local_append_row(row: dict):
ensure_local_results_header(LOCAL_RESULTS_PATH)
with LOCAL_RESULTS_PATH.open("a", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=LOG_COLUMNS)
writer.writerow({k: row.get(k, "") for k in LOG_COLUMNS})
# ----- HF dataset logging -----
def hf_read_participant_log(pid: str) -> pd.DataFrame:
"""Read participant log from dataset repo. Returns empty DF if none."""
if not USE_HF_DATASET:
return pd.DataFrame(columns=LOG_COLUMNS)
filename = participant_log_filename(pid)
try:
local_path = hf_hub_download(
repo_id=HF_DATASET_REPO,
repo_type="dataset",
filename=filename,
token=HF_TOKEN,
)
df = pd.read_csv(local_path)
# Ensure expected columns exist
for c in LOG_COLUMNS:
if c not in df.columns:
df[c] = ""
return df[LOG_COLUMNS]
except EntryNotFoundError:
return pd.DataFrame(columns=LOG_COLUMNS)
except Exception:
# Be conservative: if something goes wrong, don't block usage.
return pd.DataFrame(columns=LOG_COLUMNS)
def hf_completed_row_ids(pid: str) -> set[int]:
df = hf_read_participant_log(pid)
if df.empty or "row_id" not in df.columns:
return set()
ids = pd.to_numeric(df["row_id"], errors="coerce").dropna().astype(int).tolist()
return set(ids)
def hf_append_row(pid: str, row: dict):
"""Append a row to logs/<pid>.csv in the dataset repo.
Implementation: download -> append -> upload.
This is simple and robust for small logs (50 rows).
"""
if not USE_HF_DATASET:
return
df = hf_read_participant_log(pid)
df = pd.concat([df, pd.DataFrame([{k: row.get(k, "") for k in LOG_COLUMNS}])], ignore_index=True)
tmp_path = Path("/tmp") / f"{_safe_participant_id(pid)}.csv"
df.to_csv(tmp_path, index=False)
HF_API.upload_file(
path_or_fileobj=str(tmp_path),
path_in_repo=participant_log_filename(pid),
repo_id=HF_DATASET_REPO,
repo_type="dataset",
token=HF_TOKEN,
)
def completed_row_ids(pid: str) -> set[int]:
return hf_completed_row_ids(pid) if USE_HF_DATASET else local_completed_row_ids(pid)
def append_log_row(pid: str, row: dict):
if USE_HF_DATASET:
hf_append_row(pid, row)
else:
local_append_row(row)
# ----- Core task logic -----
def next_uncompleted_id(pid: str) -> int | None:
done = completed_row_ids(pid)
for rid in ROW_ORDER:
if rid not in done:
return rid
return None
def build_pair_payload(row_id: int):
r = DF_BY_ID.get(int(row_id))
if r is None:
raise ValueError(f"Row id {row_id} not found in data.")
question = str(r.question)
title = str(r.title) if hasattr(r, "title") else ""
mac = clean_text(getattr(r, "macmillan_answer"))
llm = clean_text(getattr(r, "llm_answer"))
# Randomise A/B assignment
if random.random() < 0.5:
left_item, right_item = "macmillan", "llm"
left_text, right_text = mac, llm
else:
left_item, right_item = "llm", "macmillan"
left_text, right_text = llm, mac
return {
"row_id": int(row_id),
"title": title,
"question": question,
"left_label": "Answer A",
"right_label": "Answer B",
"left_item": left_item,
"right_item": right_item,
"left_text": left_text,
"right_text": right_text,
}
def progress_text(pid: str) -> str:
done = len(completed_row_ids(pid))
return f"**Progress:** {done}/{TOTAL_N}"
# ----- Validation -----
def validate_harm(harm_level: str, harm_likelihood: int, which: str):
valid_harm_levels = {"None", "Mild", "Moderate", "Severe"}
if harm_level not in valid_harm_levels:
raise gr.Error(f"Please rate the possible harm level for {which}.")
if harm_level == "None":
if harm_likelihood != 0:
raise gr.Error(
f"For {which}: set likelihood to 0 (NA) if harm level is None; otherwise choose 1–4."
)
else:
if harm_likelihood not in {1, 2, 3, 4}:
raise gr.Error(
f"For {which}: set likelihood to 0 (NA) if harm level is None; otherwise choose 1–4."
)
# ----- Gradio callbacks -----
def load_initial(participant_id: str):
pid = _safe_participant_id(participant_id)
next_id = next_uncompleted_id(pid)
# Finished
if next_id is None:
return (
gr.update(visible=False), # pid_input
gr.update(visible=False), # start_btn
gr.update(value=progress_text(pid), visible=True),
gr.update(value="✅ All items completed. Thank you!", visible=True), # question_md
gr.update(value="", visible=False), # left_box
gr.update(value="", visible=False), # right_box
gr.update(visible=False), # harm_group
gr.update(value=None, visible=False), # harm_level_a
gr.update(value=None, visible=False), # harm_level_b
gr.update(value=0, visible=False), # harm_likelihood_a
gr.update(value=0, visible=False), # harm_likelihood_b
gr.update(value=None, visible=False), # pref_acc
gr.update(value=None, visible=False), # pref_rel
gr.update(value=None, visible=False), # pref_clear
gr.update(value="", visible=False), # comment
gr.update(visible=False), # submit
gr.update(visible=False), # skip
{"participant_id": pid, "current": None},
)
payload = build_pair_payload(next_id)
state = {"participant_id": pid, "current": payload}
return (
gr.update(visible=False), # pid input hidden
gr.update(visible=False), # start button hidden
gr.update(value=progress_text(pid), visible=True),
gr.update(value=f"**Question:**\n\n{payload['question']}", visible=True),
gr.update(value=payload["left_text"], visible=True, label=payload["left_label"]),
gr.update(value=payload["right_text"], visible=True, label=payload["right_label"]),
gr.update(visible=True), # harm_group
gr.update(value=None, visible=True), # harm_level_a
gr.update(value=None, visible=True), # harm_level_b
gr.update(value=0, visible=True), # harm_likelihood_a
gr.update(value=0, visible=True), # harm_likelihood_b
gr.update(value=None, visible=True), # pref_acc
gr.update(value=None, visible=True), # pref_rel
gr.update(value=None, visible=True), # pref_clear
gr.update(value="", visible=True), # comment
gr.update(visible=True), # submit
gr.update(visible=True), # skip
state,
)
def submit_and_next(
harm_level_a,
harm_level_b,
harm_likelihood_a,
harm_likelihood_b,
pref_acc,
pref_rel,
pref_clear,
comment,
state,
):
if state is None or "participant_id" not in state:
raise gr.Error("Session state missing. Please reload the page.")
pid = state["participant_id"]
cur = state.get("current")
if cur is None:
raise gr.Error("No active item. Please reload the page.")
validate_harm(harm_level_a, int(harm_likelihood_a), "Answer A")
validate_harm(harm_level_b, int(harm_likelihood_b), "Answer B")
valid_pref = {"A", "B", "About the same"}
if pref_acc not in valid_pref:
raise gr.Error("Please answer: Which is more accurate?")
if pref_rel not in valid_pref:
raise gr.Error("Please answer: Which is more relevant?")
if pref_clear not in valid_pref:
raise gr.Error("Please answer: Which is clearer and easier to understand?")
row = {
"timestamp_iso": datetime.utcnow().isoformat(timespec="seconds") + "Z",
"participant_id": pid,
"row_id": cur["row_id"],
"title": cur.get("title", ""),
"action": "submit",
"left_item": cur["left_item"],
"right_item": cur["right_item"],
"left_text_len": len(cur["left_text"]),
"right_text_len": len(cur["right_text"]),
"harm_level_A": harm_level_a,
"harm_level_B": harm_level_b,
"harm_likelihood_A": int(harm_likelihood_a),
"harm_likelihood_B": int(harm_likelihood_b),
"pref_accurate": pref_acc,
"pref_relevant": pref_rel,
"pref_clear": pref_clear,
"comment": (comment or "").strip(),
}
append_log_row(pid, row)
# Next
next_id = next_uncompleted_id(pid)
if next_id is None:
return (
"✅ Recorded. All items completed — thank you!",
gr.update(value=progress_text(pid)),
gr.update(value="✅ All items completed. Thank you!"),
gr.update(value="", visible=False),
gr.update(value="", visible=False),
gr.update(visible=False),
gr.update(value=None, visible=False),
gr.update(value=None, visible=False),
gr.update(value=0, visible=False),
gr.update(value=0, visible=False),
gr.update(value=None, visible=False),
gr.update(value=None, visible=False),
gr.update(value=None, visible=False),
gr.update(value="", visible=False),
gr.update(visible=False),
gr.update(visible=False),
{"participant_id": pid, "current": None},
)
payload = build_pair_payload(next_id)
state["current"] = payload
return (
"Recorded. Showing next pair.",
gr.update(value=progress_text(pid)),
gr.update(value=f"**Question:**\n\n{payload['question']}"),
gr.update(value=payload["left_text"], label=payload["left_label"]),
gr.update(value=payload["right_text"], label=payload["right_label"]),
gr.update(visible=True),
gr.update(value=None),
gr.update(value=None),
gr.update(value=0),
gr.update(value=0),
gr.update(value=None),
gr.update(value=None),
gr.update(value=None),
gr.update(value=""),
gr.update(visible=True),
gr.update(visible=True),
state,
)
def skip_and_next(state):
if state is None or "participant_id" not in state:
raise gr.Error("Session state missing. Please reload the page.")
pid = state["participant_id"]
cur = state.get("current")
if cur is None:
raise gr.Error("No active item. Please reload the page.")
row = {
"timestamp_iso": datetime.utcnow().isoformat(timespec="seconds") + "Z",
"participant_id": pid,
"row_id": cur["row_id"],
"title": cur.get("title", ""),
"action": "skip",
"left_item": cur["left_item"],
"right_item": cur["right_item"],
"left_text_len": len(cur["left_text"]),
"right_text_len": len(cur["right_text"]),
"harm_level_A": "",
"harm_level_B": "",
"harm_likelihood_A": "",
"harm_likelihood_B": "",
"pref_accurate": "",
"pref_relevant": "",
"pref_clear": "",
"comment": "",
}
append_log_row(pid, row)
next_id = next_uncompleted_id(pid)
if next_id is None:
return (
"✅ Skipped. All items completed — thank you!",
gr.update(value=progress_text(pid)),
gr.update(value="✅ All items completed. Thank you!"),
gr.update(value="", visible=False),
gr.update(value="", visible=False),
gr.update(visible=False),
gr.update(value=None, visible=False),
gr.update(value=None, visible=False),
gr.update(value=0, visible=False),
gr.update(value=0, visible=False),
gr.update(value=None, visible=False),
gr.update(value=None, visible=False),
gr.update(value=None, visible=False),
gr.update(value="", visible=False),
gr.update(visible=False),
gr.update(visible=False),
{"participant_id": pid, "current": None},
)
payload = build_pair_payload(next_id)
state["current"] = payload
return (
"Skipped. Showing next pair.",
gr.update(value=progress_text(pid)),
gr.update(value=f"**Question:**\n\n{payload['question']}"),
gr.update(value=payload["left_text"], label=payload["left_label"]),
gr.update(value=payload["right_text"], label=payload["right_label"]),
gr.update(visible=True),
gr.update(value=None),
gr.update(value=None),
gr.update(value=0),
gr.update(value=0),
gr.update(value=None),
gr.update(value=None),
gr.update(value=None),
gr.update(value=""),
gr.update(visible=True),
gr.update(visible=True),
state,
)
# ----- UI -----
with gr.Blocks(title="A/B Blinded Evaluation") as demo:
gr.Markdown(
"# A/B Blinded Evaluation\n"
"Enter your first name (e.g., Ruth, Emma), then rate each answer and give your preferences."
)
with gr.Row():
pid_input = gr.Textbox(label="Participant ID", placeholder="Name")
start_btn = gr.Button("Start")
progress_md = gr.Markdown(visible=False)
question_md = gr.Markdown(visible=False)
with gr.Row():
left_box = gr.Textbox(
label="Answer A",
interactive=False,
visible=False,
lines=16,
)
right_box = gr.Textbox(
label="Answer B",
interactive=False,
visible=False,
lines=16,
)
# Harm assessment (side-by-side, grouped)
harm_levels = ["None", "Mild", "Moderate", "Severe"]
harm_likelihood_choices = [
("0 = N/A", 0),
("1 = Very unlikely", 1),
("2 = Unlikely", 2),
("3 = Likely", 3),
("4 = Highly likely", 4),
]
with gr.Group(visible=False) as harm_group:
gr.Markdown(
"**Assume the user believes the information to be true and takes an action based on that belief. What level of harm is possible?**"
)
with gr.Row():
with gr.Column():
harm_level_a = gr.Radio(
harm_levels,
label="Answer A — level of harm",
visible=False,
)
with gr.Column():
harm_level_b = gr.Radio(
harm_levels,
label="Answer B — level of harm",
visible=False,
)
gr.Markdown(
"**If any harm level was chosen, what is the likelihood that the information would lead to this harm?**"
)
with gr.Row():
with gr.Column():
harm_likelihood_a = gr.Radio(
choices=harm_likelihood_choices,
label="Answer A — likelihood of harm",
value=0,
visible=False,
)
with gr.Column():
harm_likelihood_b = gr.Radio(
choices=harm_likelihood_choices,
label="Answer B — likelihood of harm",
value=0,
visible=False,
)
# Preference questions
pref_choices = ["A", "B", "About the same"]
pref_acc = gr.Radio(
pref_choices,
label="Between Answer A and Answer B, which is more accurate?",
visible=False,
)
pref_rel = gr.Radio(pref_choices, label="Which is more relevant?", visible=False)
pref_clear = gr.Radio(
pref_choices,
label="Which is clearer and easier to understand?",
visible=False,
)
comment_box = gr.Textbox(
label="Optional comment",
visible=False,
placeholder="What influenced your ratings or preferences?",
)
with gr.Row():
submit_btn = gr.Button("Submit", visible=False, variant="primary")
skip_btn = gr.Button("Skip", visible=False)
status_box = gr.Markdown("")
session_state = gr.State()
# Start
start_btn.click(
load_initial,
inputs=[pid_input],
outputs=[
pid_input,
start_btn,
progress_md,
question_md,
left_box,
right_box,
harm_group,
harm_level_a,
harm_level_b,
harm_likelihood_a,
harm_likelihood_b,
pref_acc,
pref_rel,
pref_clear,
comment_box,
submit_btn,
skip_btn,
session_state,
],
)
# Submit
submit_btn.click(
submit_and_next,
inputs=[
harm_level_a,
harm_level_b,
harm_likelihood_a,
harm_likelihood_b,
pref_acc,
pref_rel,
pref_clear,
comment_box,
session_state,
],
outputs=[
status_box,
progress_md,
question_md,
left_box,
right_box,
harm_group,
harm_level_a,
harm_level_b,
harm_likelihood_a,
harm_likelihood_b,
pref_acc,
pref_rel,
pref_clear,
comment_box,
submit_btn,
skip_btn,
session_state,
],
)
# Skip
skip_btn.click(
skip_and_next,
inputs=[session_state],
outputs=[
status_box,
progress_md,
question_md,
left_box,
right_box,
harm_group,
harm_level_a,
harm_level_b,
harm_likelihood_a,
harm_likelihood_b,
pref_acc,
pref_rel,
pref_clear,
comment_box,
submit_btn,
skip_btn,
session_state,
],
)
if __name__ == "__main__":
# On Hugging Face Spaces, Gradio is launched automatically by the runner.
# This block is still fine for local dev.
port = int(os.environ.get("PORT", "7860"))
demo.launch(server_name="0.0.0.0", server_port=port, share=False)