"""vlmin-train: QLoRA fine-tuning of Gemma 3 12B on the mixed jsonl. vlmin-train --config configs/train.yaml # start, or resume if a checkpoint exists vlmin-train --check-data # no GPU: lengths, label masking, one decoded example vlmin-train --max-steps 6 --run-name smoke # smoke test into its own run dir Resumable: checkpoints (adapter + optimizer + scheduler + RNG + data position) are written to //checkpoint-N on the Seagate drive; the newest `save_total_limit` are kept. Re-running the same command resumes from the newest one. SIGINT/SIGTERM stop cleanly: a checkpoint is saved first. A resume with a changed config (lr, batch, LoRA rank, dataset...) is refused unless --force-resume. """ import argparse import json import os import signal import sys from pathlib import Path os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID") # stable GPU indices for max_memory os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") # less fragmentation OOM import torch # noqa: E402 import torch.nn as nn # noqa: E402 import torch.nn.functional as F # noqa: E402 import yaml # noqa: E402 from torch.utils.checkpoint import checkpoint # noqa: E402 from transformers import ( # noqa: E402 AutoModelForImageTextToText, AutoProcessor, BitsAndBytesConfig, Trainer, TrainerCallback, TrainingArguments, ) from transformers.trainer_utils import get_last_checkpoint # noqa: E402 from .train_data import Collator, JsonlVLMDataset # noqa: E402 # fields that must not change between a run and its resume IDENTITY_KEYS = [ ("model_id",), ("lora", "r"), ("lora", "alpha"), ("lora", "target_regex"), ("train", "learning_rate"), ("train", "num_train_epochs"), ("train", "warmup_steps"), ("train", "lr_scheduler_type"), ("data", "max_seq_len"), ("data", "train"), ] def _get(cfg, path): for k in path: cfg = cfg[k] return cfg class GracefulStop(TrainerCallback): """First SIGINT/SIGTERM: finish the current step, save a checkpoint, exit. Second one: die.""" def __init__(self): self.requested = False for sig in (signal.SIGINT, signal.SIGTERM): signal.signal(sig, self._handler) def _handler(self, signum, frame): if self.requested: sys.exit(130) self.requested = True print(f"\n[train] signal {signum}: saving a checkpoint after this step, then stopping " "(send again to kill immediately)", flush=True) def on_step_end(self, args, state, control, **kw): if self.requested: control.should_save = True control.should_training_stop = True class GpuMemLog(TrainerCallback): def on_log(self, args, state, control, logs=None, **kw): if logs is not None and torch.cuda.is_available(): logs["gpu_peak_gb"] = round(torch.cuda.max_memory_allocated() / 2**30, 2) def check_run_identity(run_dir, cfg, force, has_checkpoint): """Write the identity file on first start; on resume refuse if anything that shapes training changed. Per-device batch / grad-accum may change freely as long as their product (samples per step) doesn't.""" p = run_dir / "run_identity.json" ident = {".".join(k): _get(cfg, k) for k in IDENTITY_KEYS} ident["train.effective_batch"] = cfg["train"]["per_device_train_batch_size"] * cfg["train"]["gradient_accumulation_steps"] if p.exists() and has_checkpoint: old = json.loads(p.read_text()) diff = {k: (old.get(k), v) for k, v in ident.items() if old.get(k) != v} if diff and not force: sys.exit("Refusing to resume: config changed since this run started (old, new):\n" + json.dumps(diff, indent=2) + "\nUse a new run_name, or --force-resume if you are sure.") else: # fresh run, or an earlier attempt that never reached a checkpoint p.write_text(json.dumps(ident, indent=2)) def check_data(train_ds, collator, tokenizer, n=8): from collections import Counter batch = collator([train_ds[i] for i in range(n)]) ids, labels = batch["input_ids"], batch["labels"] print(f"train samples kept: {len(train_ds)} (dropped as too long: {train_ds.dropped})") print("batch shapes:", {k: tuple(v.shape) for k, v in batch.items()}) print("tasks in batch:", Counter(train_ds[i]["task"] for i in range(n))) print("bos count per row:", [(r == tokenizer.bos_token_id).sum().item() for r in ids]) frac = [(l != -100).sum().item() / (m.sum().item()) for l, m in zip(labels, batch["attention_mask"])] print("fraction of tokens with loss per row:", [round(f, 3) for f in frac]) i = 0 tgt = labels[i][labels[i] != -100] print("\nTRAINED-ON text (row 0):", repr(tokenizer.decode(tgt))[:300]) print("last 6 target tokens:", tokenizer.convert_ids_to_tokens(tgt[-6:].tolist())) unmasked_img = ((labels == train_ds_image_token(tokenizer)) & (labels != -100)).sum().item() print("image tokens inside loss region (must be 0):", unmasked_img) class ChunkedLossTrainer(Trainer): """Cross-entropy without ever materialising the full (tokens x 262k vocab) logits. The model's lm_head is replaced by nn.Identity, so the forward returns hidden states; the vocab projection, logit soft-capping and loss run here in `chunk`-token slices, each recomputed in backward (checkpoint). Only answer positions are projected. Saves ~3-4 GB per sample at 2048 tokens, which is what lets 10-12 GB cards train. Same maths as the stock loss (mean over answer tokens per micro-batch).""" def __init__(self, *a, lm_weight=None, softcap=None, chunk=256, **kw): super().__init__(*a, **kw) self.lm_weight, self.softcap, self.chunk = lm_weight, softcap, chunk self.model_accepts_loss_kwargs = False # we return a per-micro-batch mean; Trainer divides by grad-accum def _piece(self, h, y): lg = (h.to(self.lm_weight.dtype) @ self.lm_weight.t()).float() if self.softcap: lg = torch.tanh(lg / self.softcap) * self.softcap return F.cross_entropy(lg, y, reduction="sum") def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None): labels = inputs.pop("labels") hidden = model(**inputs).logits # = final hidden states (lm_head is Identity) h, y = hidden[:, :-1], labels[:, 1:].to(hidden.device) keep = y != -100 n = int(keep.sum()) if n == 0: return hidden.sum() * 0 h, y = h[keep], y[keep] total = 0.0 for i in range(0, n, self.chunk): total = total + checkpoint(self._piece, h[i:i + self.chunk], y[i:i + self.chunk], use_reentrant=False) loss = total / n return (loss, None) if return_outputs else loss def prediction_step(self, model, inputs, prediction_loss_only, ignore_keys=None): with torch.no_grad(), self.compute_loss_context_manager(): loss = self.compute_loss(model, dict(inputs)).detach() return loss, None, None IMAGE_TOKEN = "" # overridden from cfg["tokens"]["image"] def train_ds_image_token(tokenizer): return tokenizer.convert_tokens_to_ids(IMAGE_TOKEN) def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--config", default="configs/train.yaml") ap.add_argument("--run-name") ap.add_argument("--output-root") ap.add_argument("--max-steps", type=int, default=-1, help="stop after N optimizer steps (smoke tests)") ap.add_argument("--train-limit", type=int, help="use a random subset of N training samples") ap.add_argument("--longest", type=int, help="memory probe: train on only the K longest samples") ap.add_argument("--no-resume", action="store_true", help="ignore existing checkpoints (the run dir must be empty)") ap.add_argument("--force-resume", action="store_true", help="resume even if the config changed") ap.add_argument("--set", action="append", default=[], metavar="KEY=VAL", help="override a config value, e.g. --set train.save_steps=2 (repeatable)") ap.add_argument("--check-data", action="store_true", help="validate data pipeline on CPU and exit") args = ap.parse_args() cfg = yaml.safe_load(Path(args.config).read_text()) for kv in args.set: key, val = kv.split("=", 1) node = cfg *parents, leaf = key.split(".") for k in parents: node = node[k] val = yaml.safe_load(val) if isinstance(val, str): # YAML 1.1 reads "5e-5" as a string try: val = float(val) except ValueError: pass node[leaf] = val if args.run_name: cfg["run_name"] = args.run_name if args.output_root: cfg["output_root"] = args.output_root mount = cfg.get("require_mount") if mount and not os.path.ismount(mount): sys.exit(f"{mount} is not mounted (did the machine reboot?). Mount it and re-run; nothing was written.") run_dir = Path(cfg["output_root"]) / cfg["run_name"] d, t = cfg["data"], cfg["train"] processor = AutoProcessor.from_pretrained(cfg["model_id"]) tok = processor.tokenizer print("[train] indexing + length-filtering datasets ...", flush=True) train_ds = JsonlVLMDataset(d["train"], tok, d["max_seq_len"], d["max_image_side"], limit=args.train_limit, seed=t["seed"], longest=args.longest) val_ds = JsonlVLMDataset(d["val"], tok, d["max_seq_len"], d["max_image_side"], limit=d["eval_samples"], seed=t["seed"]) print(f"[train] train={len(train_ds)} (dropped {train_ds.dropped} too long) val={len(val_ds)}", flush=True) tk = cfg.get("tokens", {}) global IMAGE_TOKEN IMAGE_TOKEN = tk.get("image", IMAGE_TOKEN) collator = Collator(processor, tk.get("sot", ""), tk.get("eot", "")) if args.check_data: check_data(train_ds, collator, tok) return run_dir.mkdir(parents=True, exist_ok=True) last = None if args.no_resume else get_last_checkpoint(str(run_dir)) if args.no_resume and any(run_dir.glob("checkpoint-*")): sys.exit(f"--no-resume but {run_dir} already has checkpoints; pick a new --run-name") cfg_ident = cfg if args.train_limit or args.longest: cfg_ident = {**cfg, "data": {**d, "train": f"{d['train']}#limit{args.train_limit}#longest{args.longest}"}} check_run_identity(run_dir, cfg_ident, args.force_resume, last is not None) print(f"[train] run dir: {run_dir}\n[train] {'resuming from ' + last if last else 'starting fresh'}", flush=True) from peft import LoraConfig, get_peft_model bnb = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.bfloat16, llm_int8_skip_modules=["vision_tower", "multi_modal_projector", "embed_vision", "embed_audio", "vision_embedder", "lm_head"], ) # device_map: {"": 0} = one GPU; "auto" + max_memory = layers split over several GPUs (naive pipeline) dm = cfg.get("device_map", {"": 0}) world = int(os.environ.get("WORLD_SIZE", "1")) if world > 1: # launched with torchrun: data parallel, one full model copy per GPU dm = {"": int(os.environ["LOCAL_RANK"])} kw = {"max_memory": {int(k): v for k, v in cfg["max_memory"].items()}} if cfg.get("max_memory") else {} model = AutoModelForImageTextToText.from_pretrained( cfg["model_id"], quantization_config=bnb, dtype=torch.bfloat16, attn_implementation=cfg["attn_implementation"], device_map=dm, **kw, ) print("[train] device map:", {k: v for k, v in list(getattr(model, "hf_device_map", {}).items())[:6]}, "...", flush=True) model.config.use_cache = False # chunked loss: lm_head -> Identity (forward returns hidden states), soft-cap applied in ChunkedLossTrainer stock = os.environ.get("VLMIN_STOCK_LOSS") == "1" # debug: compare against the stock full-logits loss lm_weight = model.get_output_embeddings().weight tcfg = model.config.get_text_config() softcap = getattr(tcfg, "final_logit_softcapping", None) if not stock: tcfg.final_logit_softcapping = None model.set_output_embeddings(nn.Identity()) lc = cfg["lora"] model = get_peft_model(model, LoraConfig( r=lc["r"], lora_alpha=lc["alpha"], lora_dropout=lc["dropout"], target_modules=lc["target_regex"], task_type="CAUSAL_LM", )) model.print_trainable_parameters() targs = TrainingArguments( output_dir=str(run_dir), num_train_epochs=t["num_train_epochs"], max_steps=args.max_steps, per_device_train_batch_size=t["per_device_train_batch_size"], per_device_eval_batch_size=t["per_device_eval_batch_size"], gradient_accumulation_steps=t["gradient_accumulation_steps"], learning_rate=t["learning_rate"], lr_scheduler_type=t["lr_scheduler_type"], warmup_steps=t["warmup_steps"], weight_decay=t["weight_decay"], max_grad_norm=t["max_grad_norm"], optim=t["optim"], bf16=True, gradient_checkpointing=True, gradient_checkpointing_kwargs={"use_reentrant": False}, logging_steps=t["logging_steps"], save_strategy="steps", save_steps=t["save_steps"], save_total_limit=t.get("save_total_limit"), # None = keep all eval_strategy="steps", eval_steps=t["eval_steps"], dataloader_num_workers=t["dataloader_num_workers"], dataloader_pin_memory=True, ddp_find_unused_parameters=False, remove_unused_columns=False, report_to="none", seed=t["seed"], data_seed=t["seed"], ) stopper = GracefulStop() trainer = (Trainer if stock else ChunkedLossTrainer)( model=model, args=targs, train_dataset=train_ds, eval_dataset=val_ds, data_collator=collator, callbacks=[stopper, GpuMemLog()], **({} if stock else dict(lm_weight=lm_weight, softcap=softcap)), ) trainer.train(resume_from_checkpoint=last) if stopper.requested: # stopped by signal: a checkpoint was saved; exit non-zero so a supervisor resumes instead of finishing print("[train] stopped early by signal; checkpoint saved. Re-run to resume.", flush=True) sys.exit(143) final = run_dir / "final" trainer.save_model(str(final)) processor.save_pretrained(str(final)) print(f"[train] done. adapter saved to {final}", flush=True) if __name__ == "__main__": main()