Upload folder using huggingface_hub

configs/base.yaml

@@ -9,8 +9,8 @@ model:
   pad_token: "<|fim_pad|>"     # distinct from eos <|endoftext|>; avoids infinite-gen bug.
 
 lora:
-  r: 16
-  alpha: 32                    # alpha = 2r
+  r: 32                        # raised from 16: first-error localization is a hard skill
+  alpha: 64                    # alpha = 2r
   dropout: 0.05
   target_modules: "all-linear" # attention + MLP (LoRA-Learns-Less best practice)
 

scripts/inspect_verifier.py

@@ -0,0 +1,56 @@
+"""Eyeball what Model V actually outputs — to diagnose a weak ProcessBench score.
+
+    python scripts/inspect_verifier.py --adapter runs/verifier_v --n 6 --only error
+
+Prints, per example: gold first-error index, V's parsed prediction, and the raw
+critique. Reveals whether V is collapsing to -1, mislocating, off-by-one, or
+rambling/truncating.
+"""
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+
+from mathcompose.common.chat import verifier_messages  # noqa: E402
+from mathcompose.data.schema import parse_verifier_output  # noqa: E402
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--base-id", default="Qwen/Qwen2.5-Math-1.5B-Instruct")
+    ap.add_argument("--adapter", default=None)
+    ap.add_argument("--split", default="gsm8k")
+    ap.add_argument("--n", type=int, default=6)
+    ap.add_argument("--only", choices=["error", "correct", "both"], default="error")
+    ap.add_argument("--max-new-tokens", type=int, default=768)
+    args = ap.parse_args()
+
+    from datasets import load_dataset
+    from mathcompose.eval.runners import HFRunner
+
+    ds = load_dataset("Qwen/ProcessBench", split=args.split).shuffle(seed=0)
+    runner = HFRunner(args.base_id, adapter_path=args.adapter)
+
+    shown = 0
+    for row in ds:
+        gold = int(row["label"])
+        if args.only == "error" and gold == -1:
+            continue
+        if args.only == "correct" and gold != -1:
+            continue
+        out = runner.chat(verifier_messages(row["problem"], list(row["steps"])),
+                          n=1, temperature=0.0, max_new_tokens=args.max_new_tokens)[0]
+        pred = parse_verifier_output(out).first_error_index
+        flag = "HIT" if pred == gold else "miss"
+        print(f"\n{'=' * 70}\n[{flag}] gold={gold}  pred={pred}  (#steps={len(row['steps'])})")
+        print("-" * 70)
+        print(out.strip()[:1200])
+        shown += 1
+        if shown >= args.n:
+            break
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/mathcompose/data/build_verifier_dataset.py

@@ -19,7 +19,11 @@ from ..common.parallel import thread_map
 from ..teachers import get_teacher
 from ..datagen.prm800k_loader import iter_prm800k
 from ..datagen.dedup import build_contamination_set, is_contaminated
-from ..datagen.gen_verifier_data import synthesize_verifier_completion, make_verifier_row
+from ..datagen.gen_verifier_data import (
+    synthesize_verifier_completion,
+    detect_verifier_completion,
+    make_verifier_row,
+)
 
 
 def main() -> None:
@@ -31,7 +35,11 @@ def main() -> None:
     ap.add_argument("--model", default=None, help="teacher model id override")
     ap.add_argument("--limit", type=int, default=None)
     ap.add_argument("--val-frac", type=float, default=0.05)
-    ap.add_argument("--temperature", type=float, default=0.4)
+    ap.add_argument("--mode", choices=["detect", "rationalize"], default="detect",
+                    help="detect=genuine (teacher critiques blind, keep if index matches gold); "
+                         "rationalize=teacher told the gold (teaches style, not skill)")
+    ap.add_argument("--temperature", type=float, default=0.6)
+    ap.add_argument("--max-attempts", type=int, default=2, help="detect: samples per record")
     ap.add_argument("--workers", type=int, default=8, help="concurrent teacher calls")
     ap.add_argument("--no-dedup", action="store_true")
     ap.add_argument("--seed", type=int, default=0)
@@ -51,13 +59,21 @@ def main() -> None:
         if not (banned and is_contaminated(r.problem, banned))
     ]
     print(f"synthesizing critiques for {len(records)} records "
-          f"({args.workers} workers, teacher={args.teacher}) ...")
-
-    def _one(rec):
-        completion = synthesize_verifier_completion(teacher, rec, temperature=args.temperature)
-        return make_verifier_row(rec, completion)
+          f"(mode={args.mode}, {args.workers} workers, teacher={args.teacher}) ...")
+
+    if args.mode == "detect":
+        def _one(rec):
+            comp = detect_verifier_completion(teacher, rec, temperature=args.temperature,
+                                              max_attempts=args.max_attempts)
+            return make_verifier_row(rec, comp) if comp else None
+    else:
+        def _one(rec):
+            comp = synthesize_verifier_completion(teacher, rec, temperature=args.temperature)
+            return make_verifier_row(rec, comp)
 
     rows = [r for r in thread_map(_one, records, workers=args.workers, desc="verifier") if r]
+    if args.mode == "detect" and records:
+        print(f"detect keep-rate: {len(rows)}/{len(records)} = {len(rows)/len(records):.0%}")
     random.shuffle(rows)
 
     n_val = max(1, int(len(rows) * args.val_frac)) if rows else 0

src/mathcompose/datagen/gen_verifier_data.py

@@ -64,6 +64,30 @@ def synthesize_verifier_completion(
     return f"{critique}\n\n\\boxed{{{rec.first_error_index}}}"
 
 
+def detect_verifier_completion(
+    teacher, rec: PRM800KRecord, *, temperature: float = 0.6, max_attempts: int = 2
+) -> Optional[str]:
+    """GENUINE-DETECTION recipe: the teacher critiques WITHOUT seeing the gold, and
+    we keep the critique only if its predicted first-error index matches the PRM800K
+    gold (rejection sampling). Unlike rationalization, the kept critiques contain real
+    error-finding reasoning — the signal V must learn. Returns None if no attempt hit.
+    """
+    from ..data.schema import parse_verifier_output
+
+    messages = [
+        {"role": "system", "content": VERIFIER_SYSTEM},
+        {"role": "user", "content": build_verifier_prompt(rec.problem, rec.steps)},  # no gold
+    ]
+    for _ in range(max_attempts):
+        try:
+            out = teacher.complete(messages, temperature=temperature, max_tokens=1536)
+        except Exception:
+            return None
+        if parse_verifier_output(out).first_error_index == rec.first_error_index:
+            return out.strip()
+    return None
+
+
 def make_verifier_row(rec: PRM800KRecord, completion: str) -> dict:
     """Conversational prompt-completion row (triggers TRL completion_only_loss)."""
     prompt_msgs = verifier_messages(rec.problem, rec.steps)  # system + user, no gold

src/mathcompose/eval/processbench.py

@@ -143,11 +143,20 @@ def evaluate_batched(
             runner, items, n=n, batch_size=batch_size, temperature=temperature,
             max_new_tokens=max_new_tokens, desc=f"ProcessBench/{split}",
         )
-        per_subset.append(score_subset(split, golds, preds))
-
-    avg_f1 = sum(s.f1 for s in per_subset) / len(per_subset) if per_subset else 0.0
+        score = score_subset(split, golds, preds)
+        sd = score.as_dict()
+        tot = len(preds) or 1
+        # prediction distribution — reveals mode collapse (e.g. always predicting -1)
+        sd["pred_dist"] = {
+            "all_correct(-1)": round(sum(1 for p in preds if p == -1) / tot, 3),
+            "error_idx(>=0)": round(sum(1 for p in preds if p >= 0) / tot, 3),
+            "parse_fail(-2)": round(sum(1 for p in preds if p == -2) / tot, 3),
+        }
+        per_subset.append((score, sd))
+
+    avg_f1 = sum(s.f1 for s, _ in per_subset) / len(per_subset) if per_subset else 0.0
     return {
-        "subsets": {s.subset: s.as_dict() for s in per_subset},
+        "subsets": {sd["subset"]: sd for _, sd in per_subset},
         "average_f1": avg_f1,
         "gpt4o_reference_f1": 61.9,
     }

tests/test_datagen.py

@@ -2,7 +2,10 @@
 from pathlib import Path
 
 from mathcompose.datagen.prm800k_loader import iter_prm800k
-from mathcompose.datagen.gen_verifier_data import generate_verifier_dataset
+from mathcompose.datagen.gen_verifier_data import (
+    generate_verifier_dataset,
+    detect_verifier_completion,
+)
 from mathcompose.datagen.gen_generator_data import Problem, generate_generator_dataset
 from mathcompose.teachers import get_teacher
 from mathcompose.eval.parse import first_error_index
@@ -34,6 +37,21 @@ def test_verifier_datagen_labels_authoritative_and_no_leak():
         assert "FIRST error occurs in paragraph" not in user
 
 
+def test_detect_mode_keeps_only_matching_predictions():
+    """Genuine-detection recipe: keep a critique only if the teacher's blind
+    prediction matches the PRM800K gold."""
+    recs = list(iter_prm800k(FIX))
+    err = next(r for r in recs if r.first_error_index == 1)  # "What is 2 + 2?" -> gold 1
+
+    # teacher predicts the correct index -> kept
+    good = get_teacher("dummy", fn=lambda m: "Paragraph 1: wrong. Verdict: incorrect\n\n\\boxed{1}")
+    assert detect_verifier_completion(good, err, max_attempts=1) is not None
+
+    # teacher predicts the wrong index -> rejected (None)
+    bad = get_teacher("dummy", fn=lambda m: "All good.\n\n\\boxed{-1}")
+    assert detect_verifier_completion(bad, err, max_attempts=2) is None
+
+
 def test_generator_datagen_answer_filter():
     def scripted(msgs):
         q = msgs[-1]["content"]