"""Code-execution verify loop (M1c) — with a scripted runner (no model/network).""" from mathcompose.eval.code_verify import verify_with_code from mathcompose.eval.parse import PARSE_FAIL class _ScriptedRunner: """Emits a code block first, then boxes an index after seeing execution output.""" def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512): saw_exec = any("Execution results" in m["content"] for m in messages if m["role"] == "user") if saw_exec: return ["The output shows 5, contradicting the step. \\boxed{2}"] return ["Check paragraph 2.\n```python\nprint(2+3)\n```"] class _ImmediateRunner: def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512): return ["All steps verify. \\boxed{-1}"] class _GarbageRunner: def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512): return ["hmm no answer here"] def test_executes_then_boxes(): assert verify_with_code(_ScriptedRunner(), "2+3", ["a", "2+3=4", "c"], max_rounds=3) == 2 def test_immediate_box_no_code(): assert verify_with_code(_ImmediateRunner(), "p", ["a", "b"]) == -1 def test_unparseable_returns_sentinel(): assert verify_with_code(_GarbageRunner(), "p", ["a", "b"], max_rounds=2) == PARSE_FAIL