Upload script.py with huggingface_hub

script.py

@@ -0,0 +1,546 @@
+"""
+IOL-AI Challenge 2026 — submission script (OFFLINE / Mode B).
+
+Runtime facts (Space Submission tab):
+  * T4 medium, 16 GB VRAM, Python 3.10, 30-min wall clock.
+  * NO internet: cannot pip install or download anything. Model weights must be
+    committed into THIS repo (the working dir) and loaded from ".". Only the
+    pre-installed libraries/versions are available (torch 2.4.0, transformers
+    4.44.1, accelerate 0.34.2, bitsandbytes 0.43.3, autoawq 0.2.7, pandas 2.2.2,
+    numpy 2.1.3, ...). Do NOT pin different majors of torch/transformers/numpy.
+  * Read hidden test set from /tmp/data/test.csv; write submission.csv here.
+  * pred = JSON list, one entry per numbered item, in query order.
+
+Ship the model in the repo with build_repo.py. This script loads it from "." with
+bitsandbytes 4-bit by default (or auto-detected AWQ) so it fits 16 GB. T4 has no
+bf16 -> use float16.
+
+Lineage: v4 fixed the big bug — answer COUNT comes from the model/context, never a
+query regex (that clipped matching/fill-blank problems to 1 answer). v10 added
+deterministic exact-match boosters (arithmetic-eval for numbers, bijection repair for
+matching, diacritic/gloss fidelity). v11 (this file) spends the whole 30-min wall
+SAFELY: per row, a greedy anchor + sampled self-consistency (majority vote, early stop
+on consensus) + an optional refine pass ("re-derive, check, fix", folded in as one more
+vote) — all bounded by an adaptive per-row budget + per-decode max_time + soft/hard wall
+phases that degrade to single-pass then placeholder, so it can never time out (v6 did).
+Single-sequence decode throughout (v3's batched decode OOM'd). Every row keeps an
+`explanation` (Human-Eval track). Tunable: IOL_MAX_SAMPLES / IOL_CONSENSUS / IOL_REFINE
+/ IOL_TEMPERATURE / IOL_TOP_P / IOL_MAX_NEW_TOKENS / IOL_SOFT_BUDGET_S / IOL_HARD_BUDGET_S.
+
+Local dev: set IOL_TEST_CSV to a mock file. Quantization auto-disables if there's
+no CUDA so the plumbing can be exercised on CPU with a tiny model.
+"""
+
+import os
+os.environ.setdefault("HF_HUB_OFFLINE", "1")
+os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
+
+import re
+import csv
+import json
+import time
+
+SCRIPT_START = time.time()   # ~wall-clock start; the 30-min hard kill counts from here
+
+MODEL_DIR = os.environ.get("IOL_MODEL_DIR", ".")        # weights live in the repo
+TEST_CSV = os.environ.get("IOL_TEST_CSV", "/tmp/data/test.csv")
+OUT_CSV = os.environ.get("IOL_OUT_CSV", "submission.csv")
+MAX_NEW_TOKENS = int(os.environ.get("IOL_MAX_NEW_TOKENS", "768"))
+# "4bit" (bitsandbytes), "awq" (weights already AWQ-quantized), or "fp16".
+QUANT = os.environ.get("IOL_QUANT", "4bit")
+
+# --- v11: adaptive compute — self-consistency + refine, bounded by the wall -----
+# Per row: a greedy ANCHOR decode + extra SAMPLED decodes, majority-voted per item
+# (self-consistency), stopping early on consensus so easy rows free up time for hard
+# ones; then, budget permitting, a REFINE pass re-checks the draft against the data and
+# is folded in as one more vote. Everything is gated by an ADAPTIVE per-row time budget
+# and a per-decode max_time, so we spend the whole wall but ALWAYS finish (v6 timed out
+# for lack of exactly this). Single-sequence decode throughout (no OOM).
+TEMPERATURE = float(os.environ.get("IOL_TEMPERATURE", "0.7"))
+TOP_P = float(os.environ.get("IOL_TOP_P", "0.9"))
+MAX_SAMPLES = int(os.environ.get("IOL_MAX_SAMPLES", "6"))       # hard cap of decodes/row
+CONSENSUS_FRAC = float(os.environ.get("IOL_CONSENSUS", "0.6"))  # early-stop agreement level
+DO_REFINE = os.environ.get("IOL_REFINE", "1") != "0"
+MIN_DECODE_S = float(os.environ.get("IOL_MIN_DECODE_S", "20"))  # floor for a decode's max_time
+# Wall-clock phases (seconds from SCRIPT_START; 30-min hard kill = 1800):
+SOFT_BUDGET_S = float(os.environ.get("IOL_SOFT_BUDGET_S", "1560"))  # 26.0m: full pipeline before this
+HARD_BUDGET_S = float(os.environ.get("IOL_HARD_BUDGET_S", "1710"))  # 28.5m: placeholder-fill after
+
+ANSWER_MARKER = "###ANSWERS###"
+WHY_MARKER = "###WHY###"
+
+SYSTEM_PROMPT = (
+    "You are an expert competitor at the International Linguistics Olympiad. "
+    "Each problem gives data from a language you have never seen; deduce its rules "
+    "using ONLY the data and hints in the problem, then answer EVERY sub-question.\n\n"
+    "A problem can have MANY sub-questions even when the query is one sentence: e.g. "
+    "'give the correspondences' expects one answer for EACH numbered item in the data "
+    "(often a dozen or more). Work out how many answers are required and give exactly "
+    "that many, one per item, in the order the items appear.\n\n"
+    "Reason briefly, then give your answers in EXACTLY this format:\n"
+    f"{ANSWER_MARKER}\n"
+    "1. <answer to item 1>\n"
+    "2. <answer to item 2>\n"
+    "(one numbered line per sub-question, in order)\n"
+    f"{WHY_MARKER}\n"
+    "- <the key rule or pattern you found>\n"
+    "- <the main evidence from the data that supports it>\n\n"
+    "Each answer line holds ONLY the requested form — a word, phrase, number, or "
+    "letter — with no restating of the question and no commentary. Answer in the "
+    "language and direction the query asks. For matching items give just the option "
+    "letter; for number items give digits or the written-out number as asked. Never "
+    "leave an item blank — always give your best guess.\n\n"
+    "EXACT SPELLING MATTERS: copy the exact characters, diacritics and special symbols "
+    "that appear in the data (e.g. ʼ ɨ ŋ ʂ); never swap them for similar-looking "
+    "ordinary letters. When translating INTO English, reproduce the examples' glossing "
+    "style verbatim, including person/number markers written like you_sg, you_pl.\n\n"
+    f"The lines after {WHY_MARKER} are a SHORT, human-readable explanation (1-3 bullets "
+    "a person can grasp in under a minute) — NOT your full reasoning trace."
+)
+
+# Short, low-cost per-task output reminders (the CSV tags each row with task_type).
+TASK_HINT = {
+    "translation": "This is a translation task: each answer is only the translated word/phrase.",
+    "text_to_num": "This is a number task: each answer is only digits (e.g. 42).",
+    "num_to_text": "This is a number task: each answer is only the number written in the target language's words.",
+    "match_letters": "This is a matching task: each answer is only the option letter (A, B, C, ...); give one per item in the data.",
+    "matching": "This is a matching task: each answer is only the option letter; give one per item in the data.",
+    "fill_blank": "This is a fill-in-the-blank task: each answer is only the missing form.",
+    "fill_blanks": "This is a fill-in-the-blank task: each answer is only the missing form.",
+}
+
+
+def build_messages(row):
+    """Chat messages for one problem, with a short task_type-specific reminder plus
+    two exact-match boosters: a COMPUTE line for number tasks (we evaluate it) and the
+    exact set of valid letters for matching tasks."""
+    context = (row.get("context") or "").strip()
+    query = (row.get("query") or "").strip()
+    ttype = (row.get("task_type") or "").strip().lower()
+
+    system = SYSTEM_PROMPT
+    hint = TASK_HINT.get(ttype)
+    if hint:
+        system = system + "\n\n" + hint
+
+    if ttype == "text_to_num":
+        system += (
+            f"\n\nAfter the {WHY_MARKER} bullets, add one more line exactly:\n"
+            "COMPUTE: expr1 | expr2 | ...\n"
+            "where each expr is plain arithmetic (digits, + - *, parentheses only) that "
+            "evaluates to that item's number, one per item, matching the rule you found."
+        )
+    if ttype in ("match_letters", "matching"):
+        opts = extract_letter_options(context)
+        if opts:
+            system += (f"\n\nThe ONLY valid answers are these letters: {', '.join(opts)}. "
+                       "Use no other letter.")
+
+    return [
+        {"role": "system", "content": system},
+        {"role": "user", "content": context + "\n\n" + query},
+    ]
+
+
+def detect_count(context, query):
+    """Best-effort number of sub-questions — used ONLY as a hint and a minimum pad,
+    NEVER to truncate the model's own answer list (under-producing loses items).
+    Queries usually number items ('1.'/'2.') or mark blanks ('(1)','(2)'); matching
+    queries number nothing, so fall back to the numbered items in the CONTEXT."""
+    q = re.findall(r"(?m)^\s*(\d+)[\.\)]", query)
+    if q:
+        return len(q)
+    par = re.findall(r"\((\d+)\)", query)
+    if par:
+        return len(set(par))
+    c = re.findall(r"(?m)^\s*(\d+)[\.\)]", context)
+    if c:
+        return len(c)
+    return 1
+
+
+def _clean_answer(s):
+    """Strip list markers, common 'Answer:' labels, and surrounding quotes."""
+    s = re.sub(r"^\s*(?:\d+[\.\):]|[-*•])\s*", "", s).strip()
+    s = re.sub(r"^(?:answer|ans|translation|result)\s*[:\-]\s*", "", s, flags=re.I).strip()
+    return s.strip("\"'“”‘’` ").strip()
+
+
+def parse_answers(text, min_count=1):
+    """Extract the model's FULL answer list — the count comes from the MODEL, never
+    truncated to a query heuristic (that was the v1/v2 bug: it clipped matching
+    problems' dozen answers down to 1). Prefer the ###ANSWERS### block; inside it
+    read the numbered lines; else split a trailing comma-list; else use the lines.
+    Pad up to min_count and never emit a blank."""
+    seg = text.rsplit(ANSWER_MARKER, 1)[1] if ANSWER_MARKER in text else text
+    if WHY_MARKER in seg:                       # answers live BEFORE the WHY section
+        seg = seg.split(WHY_MARKER, 1)[0]
+
+    numbered = {}
+    for m in re.finditer(r"(?m)^\s*(\d+)[\.\)]\s*(.+?)\s*$", seg):
+        numbered[int(m.group(1))] = _clean_answer(m.group(2))
+    if numbered:                                    # ordered by the model's indices
+        answers = [numbered.get(i, "") for i in range(1, max(numbered) + 1)]
+    else:
+        lines = [ln.strip() for ln in seg.splitlines() if ln.strip()]
+        comma_line = next((ln for ln in reversed(lines) if "," in ln), "")
+        if comma_line:                              # matching-style "O, D, A, ..."
+            answers = [_clean_answer(x) for x in comma_line.split(",")]
+        else:
+            answers = [_clean_answer(ln) for ln in lines]
+
+    answers = [a if a else "?" for a in answers]    # never blank (partial credit)
+    if len(answers) < min_count:
+        answers += ["?"] * (min_count - len(answers))
+    return answers if answers else ["?"]
+
+
+def _norm(s):
+    """Mirror the official scorer's normalization so voting groups answers the
+    same way the metric will (ignore case, surrounding quotes, one trailing dot)."""
+    s = " ".join((s or "").strip().split())
+    s = s.strip("\"'“”‘’")
+    if s.endswith("."):
+        s = s[:-1]
+    return s.strip().casefold()
+
+
+def vote_lists(answer_lists, min_count=1):
+    """Majority-vote per position across already-parsed answer lists (self-consistency).
+    Most common NORMALIZED form wins; returns its surface form. Ties -> earliest list.
+    '?' placeholders don't vote, so a malformed/truncated decode can't corrupt a result."""
+    from collections import Counter
+
+    n = max([min_count] + [len(a) for a in answer_lists]) if answer_lists else min_count
+    out = []
+    for i in range(n):
+        counts, surface = Counter(), {}
+        for a in answer_lists:
+            if i < len(a) and a[i] and a[i] != "?":
+                key = _norm(a[i])
+                counts[key] += 1
+                surface.setdefault(key, a[i])
+        out.append(surface[counts.most_common(1)[0][0]] if counts else "?")
+    return out
+
+
+def vote_answers(sample_texts, min_count=1):
+    """Self-consistency over raw decodes (parse each, then vote per item)."""
+    return vote_lists([parse_answers(t, min_count) for t in sample_texts], min_count)
+
+
+def _agreement(answer_lists, voted):
+    """Fraction of lists whose normalized answers equal the voted result (early-stop)."""
+    if not answer_lists:
+        return 0.0
+    vt = tuple(_norm(x) for x in voted)
+    hit = 0
+    for a in answer_lists:
+        ap = list(a) + ["?"] * (len(vt) - len(a))
+        if tuple(_norm(x) for x in ap[:len(vt)]) == vt:
+            hit += 1
+    return hit / len(answer_lists)
+
+
+def parse_explanation(text):
+    """Pull the short ###WHY### summary the model wrote (for the Human-Eval jury track).
+    Kept concise and readable; NOT the raw reasoning trace. '' if the model omitted it.
+    The internal COMPUTE: line (used only for arithmetic eval) is dropped from it."""
+    if WHY_MARKER not in text:
+        return ""
+    why = text.rsplit(WHY_MARKER, 1)[1].replace(ANSWER_MARKER, " ").strip()
+    lines = [ln.strip() for ln in why.splitlines()
+             if ln.strip() and not re.match(r"(?i)^\s*compute\s*:", ln)]
+    return "\n".join(lines[:4])[:600].strip()
+
+
+# ---- deterministic exact-match boosters (no extra model call, adapted from v5) ----
+import ast as _ast
+_ALLOWED_BINOPS = (_ast.Add, _ast.Sub, _ast.Mult)
+_NUM_NODE = getattr(_ast, "Num", None)   # pre-3.8 number node (sandbox is 3.10=Constant)
+
+
+def _safe_arithmetic(expr):
+    """Evaluate a plain +,-,* / parenthesised integer expression, else None."""
+    try:
+        tree = _ast.parse(expr.strip(), mode="eval")
+    except Exception:
+        return None
+
+    def _ev(n):
+        if isinstance(n, _ast.Expression):
+            return _ev(n.body)
+        if isinstance(n, _ast.Constant) and isinstance(n.value, (int, float)):
+            return n.value
+        if _NUM_NODE is not None and isinstance(n, _NUM_NODE):   # Python <3.8
+            return n.n
+        if isinstance(n, _ast.BinOp) and isinstance(n.op, _ALLOWED_BINOPS):
+            l, r = _ev(n.left), _ev(n.right)
+            if l is None or r is None:
+                return None
+            if isinstance(n.op, _ast.Add):
+                return l + r
+            if isinstance(n.op, _ast.Sub):
+                return l - r
+            return l * r
+        if isinstance(n, _ast.UnaryOp) and isinstance(n.op, _ast.USub):
+            v = _ev(n.operand)
+            return -v if v is not None else None
+        return None
+
+    return _ev(tree)
+
+
+def apply_compute_overrides(text, answers):
+    """text_to_num: if the model wrote 'COMPUTE: e1 | e2', evaluate each safely and
+    override that item's answer with the exact integer — kills arithmetic slips while
+    keeping the model's derived rule. Only overrides when the eval is a clean integer."""
+    m = re.search(r"(?im)^\s*COMPUTE\s*:\s*(.+)$", text)
+    if not m:
+        return answers
+    exprs = [e.strip() for e in m.group(1).split("|")]
+    out = list(answers)
+    for i, e in enumerate(exprs[:len(out)]):
+        v = _safe_arithmetic(e)
+        if v is not None and float(v).is_integer():
+            out[i] = str(int(v))
+    return out
+
+
+def extract_letter_options(context):
+    """The option labels A, B, C ... that a matching problem offers (contiguous from A)."""
+    found = set()
+    for line in context.splitlines():
+        for m in re.finditer(r"(?:^|\s)([A-Z])[.\)]\s+\S", line):
+            found.add(m.group(1))
+    if not found:
+        return None
+    letters = sorted(found)
+    if letters != [chr(ord("A") + i) for i in range(len(letters))]:
+        return None
+    return letters if 2 <= len(letters) <= 26 else None
+
+
+def repair_bijection(answers, labels):
+    """match_letters, bijection case only: keep the letters the model committed to
+    (first occurrence wins), fill duplicate/invalid/missing slots with the leftover
+    letters in order -> a guaranteed valid permutation of exactly len(labels) items."""
+    n = len(labels)
+    labels_sorted = sorted(labels)
+    picks = []
+    for i in range(n):
+        a = answers[i] if i < len(answers) else ""
+        f = re.findall(r"[A-Za-z]", a or "")
+        c = f[0].upper() if f else ""
+        picks.append(c if c in labels else "")
+    result = [None] * n
+    used = set()
+    for i in range(n):
+        if picks[i] and picks[i] not in used:
+            result[i] = picks[i]
+            used.add(picks[i])
+    missing = [l for l in labels_sorted if l not in used]
+    mi = 0
+    for i in range(n):
+        if result[i] is None:
+            result[i] = missing[mi] if mi < len(missing) else labels_sorted[0]
+            mi += 1
+    return result
+
+
+def postprocess(row, answers, raw):
+    """Apply the deterministic boosters that fit this row's task_type."""
+    ttype = (row.get("task_type") or "").strip().lower()
+    context = (row.get("context") or "")
+    if ttype == "text_to_num":
+        answers = apply_compute_overrides(raw, answers)
+    elif ttype in ("match_letters", "matching"):
+        labels = extract_letter_options(context)
+        # Only a true bijection (numbered context items == number of labels) is safe to
+        # repair; "pick the letter for item 1,2" style (few items, reused letters) is not.
+        ctx_items = len(re.findall(r"(?m)^\s*\d+\s*[.\)]", context))
+        if labels and len(labels) >= 2 and ctx_items == len(labels):
+            answers = repair_bijection(answers, labels)
+    return answers
+
+
+def default_explanation(row):
+    """Non-empty fallback so the explanation column is populated on every row."""
+    t = (row.get("task_type") or "linguistic").replace("_", " ")
+    return f"Inferred the {t} rule from the given examples and applied it to each item."
+
+
+def _already_quantized(model_dir):
+    """True if the shipped weights are pre-quantized (e.g. AWQ) — then transformers
+    auto-detects the config and we must NOT stack bitsandbytes on top."""
+    cfg = os.path.join(model_dir, "config.json")
+    try:
+        with open(cfg, encoding="utf-8") as f:
+            return "quantization_config" in json.load(f)
+    except Exception:
+        return False
+
+
+def load_model():
+    import torch
+    from transformers import AutoTokenizer, AutoModelForCausalLM
+
+    tok = AutoTokenizer.from_pretrained(MODEL_DIR)
+    if tok.pad_token_id is None:                    # only used to silence a warning
+        tok.pad_token = tok.eos_token
+    if not torch.cuda.is_available():
+        model = AutoModelForCausalLM.from_pretrained(
+            MODEL_DIR, torch_dtype=torch.float32).eval()   # CPU dev fallback
+        return tok, model
+
+    kwargs = dict(torch_dtype=torch.float16, device_map="auto")  # T4 has no bf16
+    if _already_quantized(MODEL_DIR):
+        pass  # AWQ/pre-quant: transformers reads quantization_config from config.json
+    elif QUANT == "4bit":
+        from transformers import BitsAndBytesConfig
+        kwargs["quantization_config"] = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.float16,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_use_double_quant=True,
+        )
+    model = AutoModelForCausalLM.from_pretrained(MODEL_DIR, **kwargs).eval()
+    return tok, model
+
+
+def generate_one(tok, model, messages, do_sample, max_time=None):
+    """Single-sequence decode (batch=1) — the VRAM-safe path proven in v1/v2. (v3's
+    batched multi-sequence decode OOM'd the T4 and produced an empty submission.)
+    max_time caps this one decode's wall time so the pipeline can't overrun."""
+    import torch
+
+    dev = model.device if hasattr(model, "device") else "cpu"
+    ids = tok.apply_chat_template(
+        messages, add_generation_prompt=True, return_tensors="pt").to(dev)
+    gkw = dict(max_new_tokens=MAX_NEW_TOKENS, pad_token_id=tok.pad_token_id)
+    if do_sample:
+        gkw.update(do_sample=True, temperature=TEMPERATURE, top_p=TOP_P)
+    else:
+        gkw.update(do_sample=False)
+    if max_time and max_time > 0:
+        gkw["max_time"] = float(max_time)              # MaxTimeCriteria stops the decode
+    with torch.no_grad():
+        gen = model.generate(ids, **gkw)
+    return tok.decode(gen[0][ids.shape[-1]:], skip_special_tokens=True).strip()
+
+
+def build_refine_messages(row, answers):
+    """A 'check and correct' pass: show the draft answers and ask the model to re-derive
+    the rule from the data, verify each answer, and output a corrected list (same format)."""
+    context = (row.get("context") or "").strip()
+    query = (row.get("query") or "").strip()
+    draft = "\n".join("%d. %s" % (i + 1, a) for i, a in enumerate(answers))
+    system = (
+        "You are re-checking a DRAFT solution to an International Linguistics Olympiad "
+        "problem. Re-derive the rules STRICTLY from the data, test them on every example, "
+        "then verify each draft answer and FIX any that are wrong (keep the right ones). "
+        "Output the corrected answers in EXACTLY this format:\n"
+        f"{ANSWER_MARKER}\n1. <answer 1>\n2. <answer 2>\n(one per sub-question, in order)\n"
+        f"{WHY_MARKER}\n- <the rule you used>\n- <what you changed, if anything>\n\n"
+        "Copy exact characters/diacritics; give only the requested form; never leave a blank."
+    )
+    user = "%s\n\n%s\n\nDRAFT ANSWERS:\n%s" % (context, query, draft)
+    return [{"role": "system", "content": system}, {"role": "user", "content": user}]
+
+
+def _well_formed(raw, answer_list, min_count):
+    """A refine/sample result is trustworthy enough to vote only if it hit the marker
+    and yielded at least min_count real (non-'?') answers."""
+    real = sum(1 for a in answer_list if a and a != "?")
+    return (ANSWER_MARKER in raw) and real >= min_count
+
+
+def solve_row(tok, model, row, row_deadline):
+    """Adaptive per-row compute: greedy anchor + sampled self-consistency (early stop on
+    consensus) + an optional refine vote, all bounded by row_deadline and the wall phases.
+    Returns (answers, explanation, mode)."""
+    context = (row.get("context") or "").strip()
+    query = (row.get("query") or "").strip()
+    min_count = detect_count(context, query)
+    messages = build_messages(row)
+
+    def budget():                                   # time left for THIS row's next decode
+        by_row = row_deadline - time.time()
+        by_wall = SOFT_BUDGET_S - (time.time() - SCRIPT_START)
+        return min(by_row, by_wall)
+
+    elapsed = time.time() - SCRIPT_START
+    if elapsed > HARD_BUDGET_S:                      # no time: safe placeholder, keep file whole
+        return ["?"] * min_count, default_explanation(row), "placeholder"
+    if elapsed > SOFT_BUDGET_S:                      # low time: one greedy pass only
+        raw = generate_one(tok, model, messages, False, max_time=max(MIN_DECODE_S, budget()))
+        return (postprocess(row, parse_answers(raw, min_count), raw),
+                parse_explanation(raw) or default_explanation(row), "single")
+
+    # --- full pipeline: greedy anchor, then sampled self-consistency ---
+    anchor = generate_one(tok, model, messages, False, max_time=max(MIN_DECODE_S, budget()))
+    texts = [anchor]
+    lists = [postprocess(row, parse_answers(anchor, min_count), anchor)]
+    while len(texts) < MAX_SAMPLES and budget() > MIN_DECODE_S:
+        t = generate_one(tok, model, messages, True, max_time=budget())
+        texts.append(t)
+        lists.append(postprocess(row, parse_answers(t, min_count), t))
+        if len(lists) >= 3 and _agreement(lists, vote_lists(lists, min_count)) >= CONSENSUS_FRAC:
+            break
+
+    voted = vote_lists(lists, min_count)
+    explanation = parse_explanation(anchor) or default_explanation(row)
+    mode = "vote%d" % len(texts)
+
+    # --- optional refine pass, folded in as one more vote (only if well-formed) ---
+    if DO_REFINE and budget() > MIN_DECODE_S:
+        rtext = generate_one(tok, model, build_refine_messages(row, voted), False,
+                             max_time=budget())
+        rlist = postprocess(row, parse_answers(rtext, min_count), rtext)
+        if _well_formed(rtext, rlist, min_count):
+            voted = vote_lists(lists + [rlist], min_count)
+            explanation = parse_explanation(rtext) or explanation
+            mode += "+refine"
+
+    return voted, explanation, mode
+
+
+def main():
+    tok, model = load_model()
+
+    with open(TEST_CSV, newline="", encoding="utf-8") as f:
+        rows = list(csv.DictReader(f))
+
+    # Write incrementally so a hard 30-min kill still leaves a valid partial file.
+    # 'explanation' column opts into the Human-Eval jury track (not auto-scored).
+    fout = open(OUT_CSV, "w", newline="", encoding="utf-8")
+    writer = csv.DictWriter(fout, fieldnames=["id", "pred", "explanation"])
+    writer.writeheader()
+    fout.flush()
+
+    n = len(rows)
+    for k, r in enumerate(rows):
+        rows_left = n - k
+        soft_left = SOFT_BUDGET_S - (time.time() - SCRIPT_START)
+        # Even share of the remaining soft budget; rows that finish early hand their
+        # slack to later rows (next iteration's soft_left is larger).
+        row_deadline = time.time() + max(0.0, soft_left) / max(1, rows_left)
+        try:
+            answers, explanation, mode = solve_row(tok, model, r, row_deadline)
+        except Exception as e:      # one row must never zero the whole submission
+            print("row %s failed: %r" % (r.get("id"), e), flush=True)
+            mc = detect_count((r.get("context") or ""), (r.get("query") or ""))
+            answers, explanation, mode = ["?"] * mc, default_explanation(r), "error"
+        writer.writerow({"id": r["id"],
+                         "pred": json.dumps(answers, ensure_ascii=False),
+                         "explanation": explanation})
+        fout.flush()                                    # survive a hard timeout
+        print("%d/%d [%s] elapsed=%ds" % (k + 1, n, mode, time.time() - SCRIPT_START),
+              flush=True)
+
+    fout.close()
+    print("wrote %s (%d rows)" % (OUT_CSV, n), flush=True)
+
+
+if __name__ == "__main__":
+    main()