File size: 2,069 Bytes
2291d9e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | """Eyeball what Model V actually outputs — to diagnose a weak ProcessBench score.
python scripts/inspect_verifier.py --adapter runs/verifier_v --n 6 --only error
Prints, per example: gold first-error index, V's parsed prediction, and the raw
critique. Reveals whether V is collapsing to -1, mislocating, off-by-one, or
rambling/truncating.
"""
import argparse
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from mathcompose.common.chat import verifier_messages # noqa: E402
from mathcompose.data.schema import parse_verifier_output # noqa: E402
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--base-id", default="Qwen/Qwen2.5-Math-1.5B-Instruct")
ap.add_argument("--adapter", default=None)
ap.add_argument("--split", default="gsm8k")
ap.add_argument("--n", type=int, default=6)
ap.add_argument("--only", choices=["error", "correct", "both"], default="error")
ap.add_argument("--max-new-tokens", type=int, default=768)
args = ap.parse_args()
from datasets import load_dataset
from mathcompose.eval.runners import HFRunner
ds = load_dataset("Qwen/ProcessBench", split=args.split).shuffle(seed=0)
runner = HFRunner(args.base_id, adapter_path=args.adapter)
shown = 0
for row in ds:
gold = int(row["label"])
if args.only == "error" and gold == -1:
continue
if args.only == "correct" and gold != -1:
continue
out = runner.chat(verifier_messages(row["problem"], list(row["steps"])),
n=1, temperature=0.0, max_new_tokens=args.max_new_tokens)[0]
pred = parse_verifier_output(out).first_error_index
flag = "HIT" if pred == gold else "miss"
print(f"\n{'=' * 70}\n[{flag}] gold={gold} pred={pred} (#steps={len(row['steps'])})")
print("-" * 70)
print(out.strip()[:1200])
shown += 1
if shown >= args.n:
break
return 0
if __name__ == "__main__":
raise SystemExit(main())
|