Upload folder using huggingface_hub

src/mathcompose/common/chat.py

@@ -58,6 +58,31 @@ def verifier_messages(problem: str, steps: list[str]) -> list[dict]:
     ]
 
 
+CODE_VERIFIER_SYSTEM = (
+    "You are a meticulous math grader with a Python interpreter. You are given a "
+    "problem and a candidate solution split into indexed paragraphs. Check each "
+    "paragraph; whenever a step involves arithmetic or algebra, VERIFY it by writing "
+    "a Python snippet in a ```python ... ``` block and reading the printed result "
+    "before judging. Find the FIRST paragraph containing an error."
+)
+
+_CODE_VERIFIER_INSTRUCTION = _VERIFIER_INSTRUCTION + (
+    "\n\nYou may write ```python``` blocks to check any computation; I will run them "
+    "and return the output. When you are certain, give the final index in \\boxed{{}}."
+)
+
+
+def build_code_verifier_prompt(problem: str, steps: list[str]) -> str:
+    return _CODE_VERIFIER_INSTRUCTION.format(problem=problem.strip(), tagged_steps=tag_steps(steps))
+
+
+def code_verifier_messages(problem: str, steps: list[str]) -> list[dict]:
+    return [
+        {"role": "system", "content": CODE_VERIFIER_SYSTEM},
+        {"role": "user", "content": build_code_verifier_prompt(problem, steps)},
+    ]
+
+
 def build_generator_prompt(problem: str) -> str:
     return problem.strip()
 

src/mathcompose/eval/code_verify.py

@@ -0,0 +1,59 @@
+r"""Inference-time code-execution verify loop (GenPRM-style, no retrain required).
+
+The verifier is told it may write ```python``` blocks to check arithmetic. We run
+this loop per example:
+  1. generate a critique,
+  2. if it emitted code AND hasn't boxed a final index yet, execute the code and
+     feed the outputs back as a tool turn,
+  3. repeat up to ``max_rounds``, then read the boxed first-error index.
+
+Because Qwen2.5-Math already supports tool-use, this can be run on the *existing*
+base/tuned models to test whether a calculator closes the arithmetic gap — before
+committing to a code-verified retrain.
+"""
+from __future__ import annotations
+
+from ..common.chat import code_verifier_messages
+from ..common.math_grade import extract_boxed_int
+from ..common.pyexec import extract_code_blocks, run_python
+from .parse import PARSE_FAIL
+
+
+def verify_with_code(
+    runner,
+    problem: str,
+    steps: list[str],
+    max_rounds: int = 3,
+    max_new_tokens: int = 512,
+    temperature: float = 0.0,
+    exec_timeout: float = 5.0,
+) -> int:
+    """Return the first-error index (or -1 all-correct, PARSE_FAIL if unparseable)."""
+    messages = code_verifier_messages(problem, steps)
+    last = ""
+    for round_i in range(max_rounds):
+        out = runner.chat(messages, n=1, temperature=temperature, max_new_tokens=max_new_tokens)[0]
+        last = out
+        idx = extract_boxed_int(out)
+        blocks = extract_code_blocks(out)
+
+        # Boxed answer with no pending code, or out of rounds -> done.
+        if idx is not None and (not blocks or round_i == max_rounds - 1):
+            return idx
+        if not blocks:
+            if idx is not None:
+                return idx
+            # nudge once for the boxed index
+            messages.append({"role": "assistant", "content": out})
+            messages.append({"role": "user", "content": "State the first-error index as \\boxed{index}."})
+            continue
+
+        # run the code, feed results back
+        results = [run_python(b, timeout=exec_timeout).as_feedback() for b in blocks]
+        feedback = "\n".join(f"[block {i} output]\n{r}" for i, r in enumerate(results))
+        messages.append({"role": "assistant", "content": out})
+        messages.append({"role": "user", "content":
+                         f"Execution results:\n{feedback}\n\nContinue, then give \\boxed{{index}}."})
+
+    idx = extract_boxed_int(last)
+    return idx if idx is not None else PARSE_FAIL

src/mathcompose/eval/processbench.py

@@ -116,6 +116,54 @@ def evaluate(
     }
 
 
+def evaluate_codeexec(
+    runner,
+    splits: Sequence[str] = DEFAULT_SPLITS,
+    limit: Optional[int] = None,
+    max_rounds: int = 3,
+    max_new_tokens: int = 512,
+    temperature: float = 0.0,
+    dataset_id: str = "Qwen/ProcessBench",
+) -> dict:
+    """Per-example code-execution verify loop (GenPRM-style). Slower (sequential
+    tool-use per example) — use a small --limit. Tests whether a calculator closes
+    the arithmetic gap, on any existing model."""
+    from datasets import load_dataset
+
+    from .code_verify import verify_with_code
+
+    try:
+        from tqdm.auto import tqdm
+    except Exception:  # pragma: no cover
+        def tqdm(x, **k):
+            return x
+
+    per_subset = []
+    for split in splits:
+        ds = load_dataset(dataset_id, split=split)
+        if limit is not None:
+            ds = ds.shuffle(seed=0).select(range(min(limit, len(ds))))
+        golds, preds = [], []
+        for row in tqdm(ds, desc=f"ProcessBench+code/{split}"):
+            golds.append(int(row["label"]))
+            preds.append(verify_with_code(
+                runner, row["problem"], list(row["steps"]),
+                max_rounds=max_rounds, max_new_tokens=max_new_tokens, temperature=temperature))
+        score = score_subset(split, golds, preds)
+        sd = score.as_dict()
+        tot = len(preds) or 1
+        sd["pred_dist"] = {
+            "all_correct(-1)": round(sum(1 for p in preds if p == -1) / tot, 3),
+            "error_idx(>=0)": round(sum(1 for p in preds if p >= 0) / tot, 3),
+            "parse_fail(-2)": round(sum(1 for p in preds if p == -2) / tot, 3),
+        }
+        per_subset.append((score, sd))
+
+    avg_f1 = sum(s.f1 for s, _ in per_subset) / len(per_subset) if per_subset else 0.0
+    return {"subsets": {sd["subset"]: sd for _, sd in per_subset},
+            "average_f1": avg_f1, "gpt4o_reference_f1": 61.9, "mode": "code-exec"}
+
+
 def evaluate_batched(
     runner,
     splits: Sequence[str] = DEFAULT_SPLITS,

src/mathcompose/eval/run.py

@@ -78,12 +78,17 @@ def _prompts_per_batch(a):
 
 
 def cmd_processbench(a):
-    from .processbench import evaluate_batched
     r = _runner(a.base_id, a.adapter)
-    res = evaluate_batched(r, splits=tuple(a.splits), limit=a.limit, n=a.maj_k,
-                           batch_size=_prompts_per_batch(a),
-                           temperature=(0.6 if a.maj_k > 1 else 0.0),
-                           max_new_tokens=a.max_new_tokens)
+    if getattr(a, "code_exec", False):
+        from .processbench import evaluate_codeexec
+        res = evaluate_codeexec(r, splits=tuple(a.splits), limit=a.limit,
+                                max_rounds=a.max_rounds, max_new_tokens=a.max_new_tokens)
+    else:
+        from .processbench import evaluate_batched
+        res = evaluate_batched(r, splits=tuple(a.splits), limit=a.limit, n=a.maj_k,
+                               batch_size=_prompts_per_batch(a),
+                               temperature=(0.6 if a.maj_k > 1 else 0.0),
+                               max_new_tokens=a.max_new_tokens)
     _save("processbench", a.tag, res)
 
 
@@ -154,6 +159,9 @@ def main() -> None:
     p.add_argument("--max-seqs", type=int, default=0, help="max concurrent sequences (0=auto by VRAM)")
     p.add_argument("--max-new-tokens", type=int, default=640)
     p.add_argument("--splits", nargs="+", default=["gsm8k", "math", "olympiadbench", "omnimath"])
+    p.add_argument("--code-exec", action="store_true",
+                   help="run the code-execution verify loop (GenPRM-style; per-example, slower)")
+    p.add_argument("--max-rounds", type=int, default=3, help="code-exec: max generate/execute rounds")
     p.set_defaults(func=cmd_processbench)
 
     p = sub.add_parser("prmbench")

tests/test_code_verify.py

@@ -0,0 +1,34 @@
+"""Code-execution verify loop (M1c) — with a scripted runner (no model/network)."""
+from mathcompose.eval.code_verify import verify_with_code
+from mathcompose.eval.parse import PARSE_FAIL
+
+
+class _ScriptedRunner:
+    """Emits a code block first, then boxes an index after seeing execution output."""
+    def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512):
+        saw_exec = any("Execution results" in m["content"] for m in messages if m["role"] == "user")
+        if saw_exec:
+            return ["The output shows 5, contradicting the step. \\boxed{2}"]
+        return ["Check paragraph 2.\n```python\nprint(2+3)\n```"]
+
+
+class _ImmediateRunner:
+    def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512):
+        return ["All steps verify. \\boxed{-1}"]
+
+
+class _GarbageRunner:
+    def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512):
+        return ["hmm no answer here"]
+
+
+def test_executes_then_boxes():
+    assert verify_with_code(_ScriptedRunner(), "2+3", ["a", "2+3=4", "c"], max_rounds=3) == 2
+
+
+def test_immediate_box_no_code():
+    assert verify_with_code(_ImmediateRunner(), "p", ["a", "b"]) == -1
+
+
+def test_unparseable_returns_sentinel():
+    assert verify_with_code(_GarbageRunner(), "p", ["a", "b"], max_rounds=2) == PARSE_FAIL