ml/verify.py

"""Bridge to the Rust verifier. The model's generated holes are reconstructed
into full programs and checked by execution — the SAME sandbox/verifier that
labelled the data (EVALUATION.md: one verifier, no separate harness)."""

from __future__ import annotations

import json
import subprocess

BIN = "./target/release/echo-data"


def verify_batch(candidates: list[dict], budget: int = 200_000) -> list[bool]:
    """candidates: [{"source": str, "tests": [{"input":[...],"output":...}]}].
    Returns a list of pass/fail booleans, one per candidate."""
    if not candidates:
        return []
    payload = "\n".join(json.dumps(c) for c in candidates)
    proc = subprocess.run(
        [BIN, "verify-batch", "--budget", str(budget)],
        input=payload, capture_output=True, text=True,
    )
    if proc.returncode != 0:
        raise RuntimeError(f"verify-batch failed: {proc.stderr[:500]}")
    out = []
    for line in proc.stdout.splitlines():
        if line.strip():
            out.append(json.loads(line)["pass"])
    return out