Upload folder using huggingface_hub

configs/verifier_v_7b.yaml

@@ -12,22 +12,28 @@ hub_model_id: null            # set via --hub-model-id, e.g. "42e/mathcompose-ve
 model:
   base_id: "Qwen/Qwen2.5-Math-7B-Instruct"
   max_context: 4096
+  # For the FAST path (packing), install flash-attn and set:
+  #   attn_implementation: flash_attention_2
+  attn_implementation: null
 
 quant:
-  load_in_4bit: false          # bf16 LoRA on the 80GB A100: faster than 4-bit (no dequant),
-                               # and actually uses the VRAM (14GB weights vs ~4.5GB in 4-bit)
+  load_in_4bit: false   # bf16 LoRA on the 80GB A100 (no dequant; uses the VRAM)
 
 data:
   train_path: "data/verifier/train.jsonl"   # v2 genuine-detection data
   val_path: "data/verifier/val.jsonl"
 
 train:
-  max_length: 2560   # measured: V data maxes at 2334 tokens (p99 1715) — 4096 was wasteful
-  per_device_train_batch_size: 16   # up from 4: ~4x fewer forward passes -> much faster
-  gradient_accumulation_steps: 1    # eff batch 16
-  gradient_checkpointing: true      # MUST stay on for 7B (a bug had disabled it for bf16)
-  # ~30-40GB VRAM at these settings. To use more / go faster: raise batch to 24/32.
-  # NOTE: training has no auto-OOM-recovery — if it OOMs, lower the batch.
+  max_length: 2560                  # measured: V data maxes at 2334 tokens (p99 1715)
+  per_device_train_batch_size: 2    # no-checkpoint -> a long batch caps the size; 2 is safe (~47GB)
+  gradient_accumulation_steps: 8    # eff batch 16
+  gradient_checkpointing: false     # 80GB + short data -> skip the recompute tax (was the >3hr cause)
+  optim: adamw_torch_fused
+  packing: false
+  # ~1.5 hr at these settings. FASTEST (~30-45 min) = packing, which needs FlashAttention-2:
+  #   pip install flash-attn --no-build-isolation
+  # then set model.attn_implementation: flash_attention_2 AND train.packing: true.
+  # To push batch without packing: try --batch-size 3 (may OOM on a long batch) or --precision 4bit.
 
 eval:
   processbench_splits: ["gsm8k", "math", "olympiadbench", "omnimath"]

src/mathcompose/eval/parse.py

@@ -32,7 +32,10 @@ def majority_index(samples: list[str]) -> int:
     counts = Counter(idxs)
     top = max(counts.values())
     winners = [i for i, c in counts.items() if c == top]
-    return min(winners)
+    # On a tie, surface the earliest actual error rather than defaulting to -1
+    # (only report all-correct when -1 is the *sole* top vote).
+    non_neg = [w for w in winners if w >= 0]
+    return min(non_neg) if non_neg else -1
 
 
 def p_correct_from_samples(samples: list[str]) -> float:

src/mathcompose/eval/processbench.py

@@ -33,6 +33,13 @@ def harmonic_mean(a: float, b: float) -> float:
     return 2 * a * b / (a + b)
 
 
+def _pct(sd: dict) -> dict:
+    """Present accuracies/F1 on the 0-100 scale (to match GPT-4o's 61.9)."""
+    for k in ("acc_error", "acc_correct", "f1"):
+        sd[k] = round(sd[k] * 100, 2)
+    return sd
+
+
 @dataclass
 class SubsetScore:
     subset: str
@@ -108,10 +115,10 @@ def evaluate(
             golds.append(int(row["label"]))
         per_subset.append(score_subset(split, golds, preds))
 
-    avg_f1 = sum(s.f1 for s in per_subset) / len(per_subset) if per_subset else 0.0
+    avg_f1 = 100 * sum(s.f1 for s in per_subset) / len(per_subset) if per_subset else 0.0
     return {
-        "subsets": {s.subset: s.as_dict() for s in per_subset},
-        "average_f1": avg_f1,
+        "subsets": {s.subset: _pct(s.as_dict()) for s in per_subset},
+        "average_f1": round(avg_f1, 2),
         "gpt4o_reference_f1": 61.9,
     }
 
@@ -157,9 +164,10 @@ def evaluate_codeexec(
             "error_idx(>=0)": round(sum(1 for p in preds if p >= 0) / tot, 3),
             "parse_fail(-2)": round(sum(1 for p in preds if p == -2) / tot, 3),
         }
+        _pct(sd)
         per_subset.append((score, sd))
 
-    avg_f1 = sum(s.f1 for s, _ in per_subset) / len(per_subset) if per_subset else 0.0
+    avg_f1 = round(100 * sum(s.f1 for s, _ in per_subset) / len(per_subset), 2) if per_subset else 0.0
     return {"subsets": {sd["subset"]: sd for _, sd in per_subset},
             "average_f1": avg_f1, "gpt4o_reference_f1": 61.9, "mode": "code-exec"}
 
@@ -200,9 +208,10 @@ def evaluate_batched(
             "error_idx(>=0)": round(sum(1 for p in preds if p >= 0) / tot, 3),
             "parse_fail(-2)": round(sum(1 for p in preds if p == -2) / tot, 3),
         }
+        _pct(sd)
         per_subset.append((score, sd))
 
-    avg_f1 = sum(s.f1 for s, _ in per_subset) / len(per_subset) if per_subset else 0.0
+    avg_f1 = round(100 * sum(s.f1 for s, _ in per_subset) / len(per_subset), 2) if per_subset else 0.0
     return {
         "subsets": {sd["subset"]: sd for _, sd in per_subset},
         "average_f1": avg_f1,

src/mathcompose/eval/runners.py

@@ -50,6 +50,10 @@ class HFRunner:
             from peft import PeftModel
 
             model = PeftModel.from_pretrained(model, adapter_path)
+            try:
+                model = model.merge_and_unload()  # fold LoRA into base -> faster forward
+            except Exception:
+                pass  # can't merge a quantized base; keep the adapter attached
         self.model = model.to(self.device).eval()
 
     def chat(
@@ -58,7 +62,7 @@ class HFRunner:
         n: int = 1,
         temperature: float = 0.0,
         max_new_tokens: int = 1024,
-        repetition_penalty: float = 1.1,
+        repetition_penalty: float = 1.0,   # 1.1 fights V's intentionally repetitive verdicts
     ) -> list[str]:
         """Return n decoded completions (generated text only)."""
         import torch
@@ -102,16 +106,17 @@ class HFRunner:
         messages_list: list[list[dict]],
         n: int = 1,
         temperature: float = 0.0,
-        max_new_tokens: int = 640,
-        repetition_penalty: float = 1.1,
+        max_new_tokens: int = 1024,
+        repetition_penalty: float = 1.0,
     ) -> list[list[str]]:
         """Batched generation across many prompts (left-padded). Returns, per input
         prompt, a list of n decoded completions. Far faster than looping chat()."""
         import torch
 
         tok = self.tokenizer
-        old_side = tok.padding_side
-        tok.padding_side = "left"  # required for correct batched generation
+        old_side, old_trunc = tok.padding_side, tok.truncation_side
+        tok.padding_side = "left"     # required for correct batched generation
+        tok.truncation_side = "left"  # keep the TAIL (the \boxed instruction + last steps)
         try:
             rendered = [
                 tok.apply_chat_template(m, add_generation_prompt=True, tokenize=False)
@@ -123,7 +128,7 @@ class HFRunner:
                 add_special_tokens=False,
             )
         finally:
-            tok.padding_side = old_side
+            tok.padding_side, tok.truncation_side = old_side, old_trunc
 
         input_ids = enc["input_ids"].to(self.device)
         attn = enc["attention_mask"].to(self.device)
@@ -179,7 +184,7 @@ def auto_concurrent_seqs() -> int:
 
 
 def verify_batch_predictions(runner: HFRunner, items, n: int = 1, batch_size: int = 16,
-                             temperature: float = 0.0, max_new_tokens: int = 640,
+                             temperature: float = 0.0, max_new_tokens: int = 1024,
                              desc: str = "verify") -> list[int]:
     """items: list of (problem, steps). Returns the Maj@n first-error index per item,
     using batched generation. This is the fast path used by ProcessBench/PRMBench."""

src/mathcompose/train/train.py

@@ -80,7 +80,7 @@ def build_and_train(args) -> str:
     import torch
 
     if torch.cuda.is_available():
-        torch.backends.cuda.matmul.allow_tf32 = True   # free ~1.3x matmul speedup on A100+
+        torch.set_float32_matmul_precision("high")     # TF32 matmuls (A100+): ~1.3x, current API
         torch.backends.cudnn.allow_tf32 = True
     from datasets import load_dataset
     from transformers import AutoModelForCausalLM
@@ -119,16 +119,26 @@ def build_and_train(args) -> str:
     if preset:
         print(f"[preset] {label} -> {preset}")
 
-    use_4bit = (not args.no_4bit) and cfg["quant"]["load_in_4bit"] and has_cuda
-    if cfg["quant"]["load_in_4bit"] and not has_cuda and not args.no_4bit:
-        print("[warn] 4-bit requested but no CUDA — falling back to full-precision "
-              "(CPU). Use a GPU for real QLoRA runs.")
+    prec = getattr(args, "precision", "auto")
+    if prec == "4bit":
+        want_4bit = True
+    elif prec == "bf16":
+        want_4bit = False
+    else:
+        want_4bit = bool(cfg["quant"]["load_in_4bit"])
+    if getattr(args, "no_4bit", False):
+        want_4bit = False
+    use_4bit = want_4bit and has_cuda
+    if want_4bit and not has_cuda:
+        print("[warn] 4-bit requested but no CUDA — full-precision on CPU.")
 
     tok = load_tokenizer(base_id, pad_token=cfg["model"]["pad_token"])
 
     model_kwargs = {"dtype": compute_dtype}
     if use_4bit:
         model_kwargs["quantization_config"] = build_bnb_config(cfg, compute_dtype=compute_dtype)
+    if cfg["model"].get("attn_implementation"):
+        model_kwargs["attn_implementation"] = cfg["model"]["attn_implementation"]
     model = AutoModelForCausalLM.from_pretrained(base_id, **model_kwargs)
 
     if has_cuda and compute_dtype in (torch.float16, torch.float32):
@@ -181,9 +191,19 @@ def build_and_train(args) -> str:
     if grad_ckpt and hasattr(model, "enable_input_require_grads"):
         model.enable_input_require_grads()
 
-    optim = t.get("optim", "paged_adamw_8bit") if use_4bit else "adamw_torch"
+    if not has_cuda:
+        optim = "adamw_torch"                       # CPU: bnb/fused optimizers need CUDA
+    elif use_4bit:
+        optim = t.get("optim") or "paged_adamw_8bit"
+    else:
+        optim = t.get("optim") or "adamw_torch_fused"
+    packing = bool(t.get("packing", False))
+    if packing and cfg["model"].get("attn_implementation") != "flash_attention_2":
+        print("[warn] packing WITHOUT flash_attention_2 cross-contaminates examples "
+              "(attends across packed rows). Install flash-attn and set "
+              "model.attn_implementation: flash_attention_2, or disable packing.")
     print(f"[train] epochs={epochs} batch={batch} accum={accum} (eff={batch * accum}) "
-          f"max_length={maxlen} grad_ckpt={grad_ckpt}")
+          f"max_length={maxlen} grad_ckpt={grad_ckpt} packing={packing} optim={optim}")
 
     sft_args = SFTConfig(
         output_dir=output_dir,
@@ -197,11 +217,12 @@ def build_and_train(args) -> str:
         save_steps=t.get("save_steps", 50),
         max_length=maxlen,
         completion_only_loss=t.get("completion_only_loss", True),
-        packing=t.get("packing", False),
         gradient_checkpointing=grad_ckpt,
+        packing=packing,                   # concatenate rows -> ~no padding (needs FA2; see guard)
         optim=optim,
         bf16=bf16_ok and not amp_off,
         fp16=has_cuda and not bf16_ok and not amp_off,
+        save_total_limit=1,                # don't fill ephemeral disk with checkpoints
         seed=t.get("seed", 0),
         report_to=(["wandb"] if args.wandb else "none"),
         max_steps=args.max_steps if args.max_steps else -1,
@@ -255,6 +276,8 @@ def main() -> None:
                     default="auto", help="GPU-tuned defaults (auto-detects by default)")
     ap.add_argument("--no-amp", action="store_true",
                     help="disable fp16/bf16 mixed precision (T4 fallback if the scaler errors)")
+    ap.add_argument("--precision", choices=["auto", "4bit", "bf16"], default="auto",
+                    help="override base precision: 4bit (QLoRA) or bf16 (LoRA); auto=config")
     ap.add_argument("--wandb", action="store_true")
     ap.add_argument("--push", action="store_true", help="push adapter to HF Hub")
     ap.add_argument("--hub-model-id", default=None)

tests/test_eval_plumbing.py

@@ -43,5 +43,5 @@ def test_batched_processbench_eval_runs():
 
     runner = HFRunner(TINY, max_context=512)
     res = evaluate_batched(runner, splits=["gsm8k"], limit=4, n=1, batch_size=2, max_new_tokens=6)
-    assert 0.0 <= res["average_f1"] <= 1.0
+    assert 0.0 <= res["average_f1"] <= 100.0   # now reported on the 0-100 scale
     assert "gsm8k" in res["subsets"]