Upload folder using huggingface_hub

src/mathcompose/train/train.py

@@ -118,8 +118,8 @@ def build_and_train(args) -> str:
         packing=t.get("packing", False),
         gradient_checkpointing=grad_ckpt,
         optim=optim,
-        bf16=bf16_ok,
-        fp16=(has_cuda and not bf16_ok),
+        bf16=bf16_ok and not getattr(args, "no_amp", False),
+        fp16=has_cuda and not bf16_ok and not getattr(args, "no_amp", False),
         seed=t.get("seed", 0),
         report_to=(["wandb"] if args.wandb else "none"),
         max_steps=args.max_steps if args.max_steps else -1,
@@ -164,6 +164,8 @@ def main() -> None:
     ap.add_argument("--grad-accum", type=int, default=None)
     ap.add_argument("--max-length", type=int, default=None, help="sequence length (<=4096)")
     ap.add_argument("--grad-ckpt", choices=["auto", "on", "off"], default="auto")
+    ap.add_argument("--no-amp", action="store_true",
+                    help="disable fp16/bf16 mixed precision (T4 fallback if the scaler errors)")
     ap.add_argument("--wandb", action="store_true")
     ap.add_argument("--push", action="store_true", help="push adapter to HF Hub")
     ap.add_argument("--hub-model-id", default=None)