Upload folder using huggingface_hub

configs/verifier_v_7b.yaml

@@ -22,12 +22,12 @@ data:
   val_path: "data/verifier/val.jsonl"
 
 train:
-  max_length: 4096
+  max_length: 2560   # measured: V data maxes at 2334 tokens (p99 1715) — 4096 was wasteful
   per_device_train_batch_size: 16   # up from 4: ~4x fewer forward passes -> much faster
   gradient_accumulation_steps: 1    # eff batch 16
-  gradient_checkpointing: true      # keep on (no-ckpt @ seq 4096 OOMs even the 1.5B)
-  # ~35GB VRAM at these settings. To use more / go faster: raise batch to 24 or 32
-  # (fits 80GB). NOTE: training has no auto-OOM-recovery — if it OOMs, lower it.
+  gradient_checkpointing: true      # MUST stay on for 7B (a bug had disabled it for bf16)
+  # ~30-40GB VRAM at these settings. To use more / go faster: raise batch to 24/32.
+  # NOTE: training has no auto-OOM-recovery — if it OOMs, lower the batch.
 
 eval:
   processbench_splits: ["gsm8k", "math", "olympiadbench", "omnimath"]

src/mathcompose/train/train.py

@@ -174,7 +174,12 @@ def build_and_train(args) -> str:
     elif gc_mode == "off":
         grad_ckpt = False
     else:
-        grad_ckpt = bool(t.get("gradient_checkpointing", True)) and use_4bit
+        grad_ckpt = bool(t.get("gradient_checkpointing", True))  # independent of quantization
+
+    # With a frozen (esp. non-kbit) base + LoRA, checkpointing needs input grads
+    # enabled so gradients reach the adapters.
+    if grad_ckpt and hasattr(model, "enable_input_require_grads"):
+        model.enable_input_require_grads()
 
     optim = t.get("optim", "paged_adamw_8bit") if use_4bit else "adamw_torch"
     print(f"[train] epochs={epochs} batch={batch} accum={accum} (eff={batch * accum}) "