#!/usr/bin/env python3
"""Create Prolific-ready Potato data for the math study (MathVision + OrcaMath).
Adapted from the programming (MBPP) data-preparation script. It produces:
- one public Potato-ready CSV with all rows and role/subset metadata
- one public subset manifest for assigning Prolific participants to subsets
- one private gold annotation JSON for training feedback, testing, and controls
- one private local CSV that additionally contains the training feedback
The two math datasets are mixed in every role:
- training and testing: pilot dialogues from both datasets (gold = pilot labels)
- control pairs: the remaining pilot dialogues, balanced across datasets and
mixed inside every subset
- main annotation: MathVision Verified 100 + OrcaMath Verified 100, split into
blocks of 5 dialogues (alternately 3+2 and 2+3 from the two datasets)
Subsets (one participant each) chain the blocks: subset i = block i + block
i+1, wrapping around. Every subset has 10 dialogues (5 per dataset); its first
5 are shared with the previous subset and its last 5 with the next one, so
every dialogue is annotated by two different participants.
MathVision rows show the problem image, served by the Space from
`my-annotation-task/images/mathvision/` (see tools/fetch_mathvision_images.py).
The public CSV intentionally does not include gold labels.
"""
from __future__ import annotations
import argparse
import csv
import html
import json
import random
import re
from collections import Counter
from pathlib import Path
from typing import Any
DIMENSION_COLUMNS = [
"Content Correctness",
"Learner-State Assessment",
"Issue Localization",
"Disclosure Appropriateness",
"Providing Guidance",
"Coherence",
"Actionability",
"Clarity",
"Conciseness",
"Humanness",
]
ANNOTATION_KEY_ALIASES = {
"Human-likeness": "Humanness",
"Human-likeness ": "Humanness",
}
VALID_LABELS = {"Yes", "To some extent", "No"}
# Raw model keys as they appear in the JSON files -> (slug, display name).
# Responses from models missing here are rejected, so every model is included.
MODELS = {
# MathVision
"Claude Opus 4.6": ("claude_opus46", "Claude Opus 4.6"),
"Gemini 3.1 Pro": ("gemini31_pro", "Gemini 3.1 Pro"),
"GPT-5.4 2026-03-05": ("gpt54", "GPT-5.4"),
"Llama 3.2 11B Vision": ("llama32_11b_vision", "Llama 3.2 11B Vision"),
"Llama 4 Maverick": ("llama4_maverick", "Llama 4 Maverick"),
"Ministral 14B": ("ministral_14b", "Ministral 14B"),
"Phi-4 Reasoning Vision 15B": ("phi4_reasoning_vision_15b", "Phi-4 Reasoning Vision 15B"),
# OrcaMath
"ClaudeOpus46": ("claude_opus46", "Claude Opus 4.6"),
"Gemini31Pro": ("gemini31_pro", "Gemini 3.1 Pro"),
"GPT5_4": ("gpt54", "GPT-5.4"),
"GPT54Mini": ("gpt54_mini", "GPT-5.4 Mini"),
"Llama318B": ("llama31_8b", "Llama 3.1 8B"),
"Llama3370B": ("llama33_70b", "Llama 3.3 70B"),
"Mathstral7B": ("mathstral_7b", "Mathstral 7B"),
"DeepSeekMath7B": ("deepseekmath_7b", "DeepSeekMath 7B"),
}
MODEL_ORDER = list(MODELS)
MATHVISION = "MathVision"
ORCAMATH = "OrcaMath"
DATASETS = [MATHVISION, ORCAMATH]
# Curated training/testing files (labels + per-dimension explanations).
# MathVision: the CSV covers 4 pilot dialogues, split at random (seeded) 2/2 into
# training and testing. OrcaMath: the CSV holds the training dialogues; testing
# dialogues are drawn at random from the other fully labelled pilot dialogues.
MATHVISION_EXPLANATIONS_CSV = "MathVision (train+test) Explanations - 01_06_08_13.csv"
ORCAMATH_EXPLANATIONS_CSV = "math_annotator_train&test_sets - math_annotator_training_set.csv"
PRETEST_DIALOGS_PER_DATASET = 2
FLAG_COLUMNS = ["training", "testing", "annotation", "control"]
TRAINING_FEEDBACK_COLUMN = "training_feedback_json"
MISSING_TEXT_VALUES = {"", "none", "null", "n/a", "na", "nan"}
IMAGE_URL_PREFIX = "/mathvision_images/"
INSTRUCTION_TEXT = (
"Please read the following dialogue and annotate the tutor's last response "
"according to the selected quality criteria."
)
GUIDELINES_URL = "https://drive.google.com/file/d/16E43zqGCV8m4X3JtDEvjwmwJg98-7XLz/view?usp=sharing"
GUIDELINES_LINK_TEXT = "guidelines with dimension explanations and examples"
TEXT_TEMPLATE = (
"{instruction_text}\n"
"Guidelines with dimension explanations and examples: {guidelines_url}\n\n"
"Dialogue context:\n"
"{dialog_context}\n\n"
"{image_line}"
"Reference (correct) solution:\n"
"{correct_solution}\n\n"
"Failure reason:\n"
"{failure_reason}\n\n"
"Next tutor response:\n"
"{tutor_response}"
)
CODE_BLOCK_STYLE = (
"background-color: #f6f8fa; border: 1px solid #d0d7de; border-radius: 6px; "
"padding: 10px; margin: 8px 0 12px 0; font-family: monospace; "
"font-size: 14px; line-height: 1.45;"
)
PUBLIC_COLUMNS = [
"id",
"dialog_id",
"dataset",
"source",
"source_file",
"role",
"annotation_scope",
"subset_id",
"control_pair_id",
"task_id",
"mutant_id",
"mathvision_instance_id",
"image_url",
"model_name",
"model",
"sample_id_one",
"sample_id_two",
"is_concise_in_prompt",
"dialog_context",
"failure_reason",
"correct_solution",
"tutor_response",
"text2show",
"text2show_html",
"training",
"testing",
"annotation",
"control",
TRAINING_FEEDBACK_COLUMN,
]
def resolve_app_root(start: Path) -> Path:
for candidate in [start, *start.parents]:
if (candidate / "my-annotation-task" / "data").exists():
return candidate
raise FileNotFoundError("Could not find app root containing my-annotation-task/data.")
def clean_id_part(value: Any) -> str:
text = "" if value is None else str(value).strip()
if text.endswith(".0"):
try:
return str(int(float(text)))
except ValueError:
return text
return text
def is_missing_text(value: Any) -> bool:
return value is None or str(value).strip().lower() in MISSING_TEXT_VALUES
def normalize_label(value: Any) -> str:
text = "" if value is None else str(value).strip()
lowered = text.lower()
if lowered == "yes":
return "Yes"
if lowered in {"to some extent", "some extent", "partial", "partially"}:
return "To some extent"
if lowered == "no":
return "No"
return text
def read_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="", encoding="utf-8") as handle:
writer = csv.DictWriter(handle, fieldnames=PUBLIC_COLUMNS, lineterminator="\n", extrasaction="ignore")
writer.writeheader()
writer.writerows(rows)
def write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def labels_from_annotation(annotation: Any) -> dict[str, str]:
labels = {dimension: "" for dimension in DIMENSION_COLUMNS}
if not isinstance(annotation, dict):
return labels
for key, value in annotation.items():
normalized_key = ANNOTATION_KEY_ALIASES.get(key, key)
if normalized_key in labels:
labels[normalized_key] = normalize_label(value)
return labels
def labels_complete(labels: dict[str, str]) -> bool:
return all(labels.get(dimension) in VALID_LABELS for dimension in DIMENSION_COLUMNS)
# --------------------------------------------------------------------------
# Rendering: markdown-ish text with LaTeX math.
#
# Potato's HTML sanitizer escapes all text between tags again, so text is
# escaped here once and decoded once more in the browser (see config.yaml),
# after which KaTeX renders the explicit \( \) and \[ \] delimiters emitted
# below. Dollar-delimited math is converted with Pandoc's rule so currency
# amounts such as "$5 and $10" stay plain text.
# --------------------------------------------------------------------------
MATH_PLACEHOLDER = "\x00MATH{}\x00"
MATH_PLACEHOLDER_RE = re.compile("\x00MATH(\\d+)\x00")
DISPLAY_MATH_PATTERNS = [
re.compile(r"\$\$(.+?)\$\$", re.DOTALL),
re.compile(r"\\\[(.+?)\\\]", re.DOTALL),
re.compile(r"(\\begin\{([a-zA-Z*]+)\}.+?\\end\{\2\})", re.DOTALL),
]
INLINE_MATH_PATTERNS = [
re.compile(r"\\\((.+?)\\\)", re.DOTALL),
re.compile(r"(? tuple[str, list[tuple[str, bool]]]:
"""Replace math segments with placeholders; returns (text, [(tex, is_display)])."""
segments: list[tuple[str, bool]] = []
def replacer(is_display: bool, whole_env: bool = False):
def replace(match: re.Match) -> str:
tex = match.group(1)
segments.append((tex, is_display))
placeholder = MATH_PLACEHOLDER.format(len(segments) - 1)
return f"\n{placeholder}\n" if is_display else placeholder
return replace
for pattern in DISPLAY_MATH_PATTERNS:
text = pattern.sub(replacer(True), text)
for pattern in INLINE_MATH_PATTERNS:
text = pattern.sub(replacer(False), text)
return text.replace("\\$", "$"), segments
def math_html(tex: str, is_display: bool) -> str:
escaped = html.escape(tex.strip(), quote=False)
if is_display:
return f'
\\[{escaped}\\]
'
return f'\\({escaped}\\)'
def inline_markup(text: str, segments: list[tuple[str, bool]]) -> str:
escaped = html.escape(str(text), quote=False)
marked = re.sub(r"`([^`\n]+)`", lambda m: f"{m.group(1)}", escaped)
marked = re.sub(r"\*\*(.+?)\*\*", lambda m: f"{m.group(1)}", marked)
marked = re.sub(r"(?{m.group(1)}", marked)
marked = re.sub(r"(?{m.group(1)}", marked)
return MATH_PLACEHOLDER_RE.sub(lambda m: math_html(*segments[int(m.group(1))]), marked)
def table_html(lines: list[str], segments: list[tuple[str, bool]]) -> str:
rows = []
for line in lines:
if TABLE_SEPARATOR_RE.match(line):
continue
cells = [cell.strip() for cell in line.strip().strip("|").split("|")]
tag = "th" if not rows else "td"
rows.append("
" + "".join(f"<{tag}>{inline_markup(cell, segments)}{tag}>" for cell in cells) + "
")
return '
' + "".join(rows) + "
"
def rich_html(text: Any) -> str:
"""Render markdown-ish text with math as sanitizer-safe HTML."""
source, segments = protect_math(str(text or "").replace("\r\n", "\n"))
lines = source.split("\n")
parts: list[str] = []
paragraph: list[str] = []
list_tag = ""
list_items: list[str] = []
list_start = 1
def flush_paragraph() -> None:
if paragraph:
parts.append("
" + " ".join(inline_markup(line, segments) for line in paragraph) + "
")
paragraph.clear()
def flush_list() -> None:
nonlocal list_tag
if list_tag:
start = f' start="{list_start}"' if list_tag == "ol" and list_start != 1 else ""
parts.append(f"<{list_tag}{start}>" + "".join(f"
{item}
" for item in list_items) + f"{list_tag}>")
list_items.clear()
list_tag = ""
def flush_all() -> None:
flush_paragraph()
flush_list()
index = 0
while index < len(lines):
line = lines[index]
stripped = line.strip()
if FENCE_RE.match(line):
flush_all()
code_lines = []
index += 1
while index < len(lines) and not FENCE_RE.match(lines[index]):
code_lines.append(lines[index])
index += 1
code = MATH_PLACEHOLDER_RE.sub(
lambda m: segments[int(m.group(1))][0], "\n".join(code_lines)
)
parts.append(
f'
"
)
index += 1
continue
if not stripped:
flush_paragraph()
if list_tag:
# A blank line inside a list only ends it if the next line is not a list item.
next_line = next((l for l in lines[index + 1 :] if l.strip()), "")
if not ((list_tag == "ul" and BULLET_RE.match(next_line)) or (list_tag == "ol" and NUMBERED_RE.match(next_line))):
flush_list()
index += 1
continue
display = DISPLAY_ONLY_RE.match(line)
if display:
flush_paragraph()
rendered = math_html(*segments[int(display.group(1))])
if list_tag and list_items:
list_items[-1] += rendered
else:
flush_list()
parts.append(rendered)
index += 1
continue
heading = HEADING_RE.match(line)
if heading:
flush_all()
level = min(6, max(4, len(heading.group(1)) + 2))
parts.append(f"{inline_markup(heading.group(2), segments)}")
index += 1
continue
if RULE_RE.match(line):
flush_all()
parts.append("")
index += 1
continue
if TABLE_RE.match(line):
flush_all()
table_lines = []
while index < len(lines) and TABLE_RE.match(lines[index]):
table_lines.append(lines[index])
index += 1
parts.append(table_html(table_lines, segments))
continue
bullet = BULLET_RE.match(line)
numbered = NUMBERED_RE.match(line)
if bullet or numbered:
flush_paragraph()
tag = "ul" if bullet else "ol"
if list_tag != tag:
flush_list()
list_tag = tag
list_start = int(numbered.group(1)) if numbered else 1
list_items.append(inline_markup(bullet.group(1) if bullet else numbered.group(2), segments))
index += 1
continue
if list_tag and list_items and line[:1].isspace():
list_items[-1] += " " + inline_markup(stripped, segments)
index += 1
continue
flush_list()
paragraph.append(stripped)
index += 1
flush_all()
return '
' + "\n".join(parts) + "
"
def split_dialog_turns(text: str) -> list[tuple[str, str]]:
"""Split 'Tutor: ...' / 'Student: ...' conversation text into turns."""
turns: list[tuple[str, str]] = []
for match in re.finditer(r"(?ms)^(Tutor|Student):[ \t]*(.*?)(?=^(?:Tutor|Student):|\Z)", str(text)):
turns.append((match.group(1), match.group(2).strip()))
if not turns:
turns.append(("Tutor", str(text).strip()))
return turns
def make_instruction_html() -> str:
return (
f'
']
image_placed = False
for speaker, body in split_dialog_turns(str(row["dialog_context"])):
body_html = rich_html(body)
if row.get("image_url") and speaker == "Tutor" and not image_placed:
body_html += image_html(str(row["image_url"]))
image_placed = True
dialog_parts.append(utterance_html(speaker, body_html, speaker.lower()))
if row.get("image_url") and not image_placed:
dialog_parts.insert(1, image_html(str(row["image_url"])))
dialog_parts.append(utterance_html("Tutor", rich_html(row["tutor_response"]), "tutor"))
side_sections = [
'
',
'
Reference solution
',
rich_html(row["correct_solution"]),
"
",
]
if row["failure_reason"]:
side_sections += [
'