tests/test_code_verify.py

"""Code-execution verify loop (M1c) — with a scripted runner (no model/network)."""
from mathcompose.eval.code_verify import verify_with_code
from mathcompose.eval.parse import PARSE_FAIL


class _ScriptedRunner:
    """Emits a code block first, then boxes an index after seeing execution output."""
    def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512):
        saw_exec = any("Execution results" in m["content"] for m in messages if m["role"] == "user")
        if saw_exec:
            return ["The output shows 5, contradicting the step. \\boxed{2}"]
        return ["Check paragraph 2.\n```python\nprint(2+3)\n```"]


class _ImmediateRunner:
    def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512):
        return ["All steps verify. \\boxed{-1}"]


class _GarbageRunner:
    def chat(self, messages, n=1, temperature=0.0, max_new_tokens=512):
        return ["hmm no answer here"]


def test_executes_then_boxes():
    assert verify_with_code(_ScriptedRunner(), "2+3", ["a", "2+3=4", "c"], max_rounds=3) == 2


def test_immediate_box_no_code():
    assert verify_with_code(_ImmediateRunner(), "p", ["a", "b"]) == -1


def test_unparseable_returns_sentinel():
    assert verify_with_code(_GarbageRunner(), "p", ["a", "b"], max_rounds=2) == PARSE_FAIL