Image-Text-to-Text
PEFT
Safetensors
gemma4
lora
qlora
ocr
indic
vision-language
drishti / code /train.py
srivassid's picture
Gemma 4 12B Indic OCR/caption LoRA v1: adapter, card, results, code
9df0eb7 verified
Raw History Blame Contribute Delete
14.7 kB
"""vlmin-train: QLoRA fine-tuning of Gemma 3 12B on the mixed jsonl.
vlmin-train --config configs/train.yaml # start, or resume if a checkpoint exists
vlmin-train --check-data # no GPU: lengths, label masking, one decoded example
vlmin-train --max-steps 6 --run-name smoke # smoke test into its own run dir
Resumable: checkpoints (adapter + optimizer + scheduler + RNG + data position) are written to
<output_root>/<run_name>/checkpoint-N on the Seagate drive; the newest `save_total_limit` are kept. Re-running the same
command resumes from the newest one. SIGINT/SIGTERM stop cleanly: a checkpoint is saved first.
A resume with a changed config (lr, batch, LoRA rank, dataset...) is refused unless --force-resume.
"""
import argparse
import json
import os
import signal
import sys
from pathlib import Path
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID") # stable GPU indices for max_memory
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") # less fragmentation OOM
import torch # noqa: E402
import torch.nn as nn # noqa: E402
import torch.nn.functional as F # noqa: E402
import yaml # noqa: E402
from torch.utils.checkpoint import checkpoint # noqa: E402
from transformers import ( # noqa: E402
AutoModelForImageTextToText, AutoProcessor, BitsAndBytesConfig, Trainer, TrainerCallback,
TrainingArguments,
)
from transformers.trainer_utils import get_last_checkpoint # noqa: E402
from .train_data import Collator, JsonlVLMDataset # noqa: E402
# fields that must not change between a run and its resume
IDENTITY_KEYS = [
("model_id",), ("lora", "r"), ("lora", "alpha"), ("lora", "target_regex"),
("train", "learning_rate"), ("train", "num_train_epochs"),
("train", "warmup_steps"), ("train", "lr_scheduler_type"), ("data", "max_seq_len"), ("data", "train"),
]
def _get(cfg, path):
for k in path:
cfg = cfg[k]
return cfg
class GracefulStop(TrainerCallback):
"""First SIGINT/SIGTERM: finish the current step, save a checkpoint, exit. Second one: die."""
def __init__(self):
self.requested = False
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, self._handler)
def _handler(self, signum, frame):
if self.requested:
sys.exit(130)
self.requested = True
print(f"\n[train] signal {signum}: saving a checkpoint after this step, then stopping "
"(send again to kill immediately)", flush=True)
def on_step_end(self, args, state, control, **kw):
if self.requested:
control.should_save = True
control.should_training_stop = True
class GpuMemLog(TrainerCallback):
def on_log(self, args, state, control, logs=None, **kw):
if logs is not None and torch.cuda.is_available():
logs["gpu_peak_gb"] = round(torch.cuda.max_memory_allocated() / 2**30, 2)
def check_run_identity(run_dir, cfg, force, has_checkpoint):
"""Write the identity file on first start; on resume refuse if anything that shapes training changed.
Per-device batch / grad-accum may change freely as long as their product (samples per step) doesn't."""
p = run_dir / "run_identity.json"
ident = {".".join(k): _get(cfg, k) for k in IDENTITY_KEYS}
ident["train.effective_batch"] = cfg["train"]["per_device_train_batch_size"] * cfg["train"]["gradient_accumulation_steps"]
if p.exists() and has_checkpoint:
old = json.loads(p.read_text())
diff = {k: (old.get(k), v) for k, v in ident.items() if old.get(k) != v}
if diff and not force:
sys.exit("Refusing to resume: config changed since this run started (old, new):\n"
+ json.dumps(diff, indent=2) + "\nUse a new run_name, or --force-resume if you are sure.")
else: # fresh run, or an earlier attempt that never reached a checkpoint
p.write_text(json.dumps(ident, indent=2))
def check_data(train_ds, collator, tokenizer, n=8):
from collections import Counter
batch = collator([train_ds[i] for i in range(n)])
ids, labels = batch["input_ids"], batch["labels"]
print(f"train samples kept: {len(train_ds)} (dropped as too long: {train_ds.dropped})")
print("batch shapes:", {k: tuple(v.shape) for k, v in batch.items()})
print("tasks in batch:", Counter(train_ds[i]["task"] for i in range(n)))
print("bos count per row:", [(r == tokenizer.bos_token_id).sum().item() for r in ids])
frac = [(l != -100).sum().item() / (m.sum().item()) for l, m in zip(labels, batch["attention_mask"])]
print("fraction of tokens with loss per row:", [round(f, 3) for f in frac])
i = 0
tgt = labels[i][labels[i] != -100]
print("\nTRAINED-ON text (row 0):", repr(tokenizer.decode(tgt))[:300])
print("last 6 target tokens:", tokenizer.convert_ids_to_tokens(tgt[-6:].tolist()))
unmasked_img = ((labels == train_ds_image_token(tokenizer)) & (labels != -100)).sum().item()
print("image tokens inside loss region (must be 0):", unmasked_img)
class ChunkedLossTrainer(Trainer):
"""Cross-entropy without ever materialising the full (tokens x 262k vocab) logits.
The model's lm_head is replaced by nn.Identity, so the forward returns hidden states; the vocab projection,
logit soft-capping and loss run here in `chunk`-token slices, each recomputed in backward (checkpoint).
Only answer positions are projected. Saves ~3-4 GB per sample at 2048 tokens, which is what lets 10-12 GB
cards train. Same maths as the stock loss (mean over answer tokens per micro-batch)."""
def __init__(self, *a, lm_weight=None, softcap=None, chunk=256, **kw):
super().__init__(*a, **kw)
self.lm_weight, self.softcap, self.chunk = lm_weight, softcap, chunk
self.model_accepts_loss_kwargs = False # we return a per-micro-batch mean; Trainer divides by grad-accum
def _piece(self, h, y):
lg = (h.to(self.lm_weight.dtype) @ self.lm_weight.t()).float()
if self.softcap:
lg = torch.tanh(lg / self.softcap) * self.softcap
return F.cross_entropy(lg, y, reduction="sum")
def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
labels = inputs.pop("labels")
hidden = model(**inputs).logits # = final hidden states (lm_head is Identity)
h, y = hidden[:, :-1], labels[:, 1:].to(hidden.device)
keep = y != -100
n = int(keep.sum())
if n == 0:
return hidden.sum() * 0
h, y = h[keep], y[keep]
total = 0.0
for i in range(0, n, self.chunk):
total = total + checkpoint(self._piece, h[i:i + self.chunk], y[i:i + self.chunk], use_reentrant=False)
loss = total / n
return (loss, None) if return_outputs else loss
def prediction_step(self, model, inputs, prediction_loss_only, ignore_keys=None):
with torch.no_grad(), self.compute_loss_context_manager():
loss = self.compute_loss(model, dict(inputs)).detach()
return loss, None, None
IMAGE_TOKEN = "<image_soft_token>" # overridden from cfg["tokens"]["image"]
def train_ds_image_token(tokenizer):
return tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--config", default="configs/train.yaml")
ap.add_argument("--run-name")
ap.add_argument("--output-root")
ap.add_argument("--max-steps", type=int, default=-1, help="stop after N optimizer steps (smoke tests)")
ap.add_argument("--train-limit", type=int, help="use a random subset of N training samples")
ap.add_argument("--longest", type=int, help="memory probe: train on only the K longest samples")
ap.add_argument("--no-resume", action="store_true", help="ignore existing checkpoints (the run dir must be empty)")
ap.add_argument("--force-resume", action="store_true", help="resume even if the config changed")
ap.add_argument("--set", action="append", default=[], metavar="KEY=VAL",
help="override a config value, e.g. --set train.save_steps=2 (repeatable)")
ap.add_argument("--check-data", action="store_true", help="validate data pipeline on CPU and exit")
args = ap.parse_args()
cfg = yaml.safe_load(Path(args.config).read_text())
for kv in args.set:
key, val = kv.split("=", 1)
node = cfg
*parents, leaf = key.split(".")
for k in parents:
node = node[k]
val = yaml.safe_load(val)
if isinstance(val, str): # YAML 1.1 reads "5e-5" as a string
try:
val = float(val)
except ValueError:
pass
node[leaf] = val
if args.run_name:
cfg["run_name"] = args.run_name
if args.output_root:
cfg["output_root"] = args.output_root
mount = cfg.get("require_mount")
if mount and not os.path.ismount(mount):
sys.exit(f"{mount} is not mounted (did the machine reboot?). Mount it and re-run; nothing was written.")
run_dir = Path(cfg["output_root"]) / cfg["run_name"]
d, t = cfg["data"], cfg["train"]
processor = AutoProcessor.from_pretrained(cfg["model_id"])
tok = processor.tokenizer
print("[train] indexing + length-filtering datasets ...", flush=True)
train_ds = JsonlVLMDataset(d["train"], tok, d["max_seq_len"], d["max_image_side"], limit=args.train_limit, seed=t["seed"], longest=args.longest)
val_ds = JsonlVLMDataset(d["val"], tok, d["max_seq_len"], d["max_image_side"], limit=d["eval_samples"], seed=t["seed"])
print(f"[train] train={len(train_ds)} (dropped {train_ds.dropped} too long) val={len(val_ds)}", flush=True)
tk = cfg.get("tokens", {})
global IMAGE_TOKEN
IMAGE_TOKEN = tk.get("image", IMAGE_TOKEN)
collator = Collator(processor, tk.get("sot", "<start_of_turn>"), tk.get("eot", "<end_of_turn>"))
if args.check_data:
check_data(train_ds, collator, tok)
return
run_dir.mkdir(parents=True, exist_ok=True)
last = None if args.no_resume else get_last_checkpoint(str(run_dir))
if args.no_resume and any(run_dir.glob("checkpoint-*")):
sys.exit(f"--no-resume but {run_dir} already has checkpoints; pick a new --run-name")
cfg_ident = cfg
if args.train_limit or args.longest:
cfg_ident = {**cfg, "data": {**d, "train": f"{d['train']}#limit{args.train_limit}#longest{args.longest}"}}
check_run_identity(run_dir, cfg_ident, args.force_resume, last is not None)
print(f"[train] run dir: {run_dir}\n[train] {'resuming from ' + last if last else 'starting fresh'}", flush=True)
from peft import LoraConfig, get_peft_model
bnb = BitsAndBytesConfig(
load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.bfloat16,
llm_int8_skip_modules=["vision_tower", "multi_modal_projector", "embed_vision", "embed_audio", "vision_embedder", "lm_head"],
)
# device_map: {"": 0} = one GPU; "auto" + max_memory = layers split over several GPUs (naive pipeline)
dm = cfg.get("device_map", {"": 0})
world = int(os.environ.get("WORLD_SIZE", "1"))
if world > 1: # launched with torchrun: data parallel, one full model copy per GPU
dm = {"": int(os.environ["LOCAL_RANK"])}
kw = {"max_memory": {int(k): v for k, v in cfg["max_memory"].items()}} if cfg.get("max_memory") else {}
model = AutoModelForImageTextToText.from_pretrained(
cfg["model_id"], quantization_config=bnb, dtype=torch.bfloat16,
attn_implementation=cfg["attn_implementation"], device_map=dm, **kw,
)
print("[train] device map:", {k: v for k, v in list(getattr(model, "hf_device_map", {}).items())[:6]}, "...", flush=True)
model.config.use_cache = False
# chunked loss: lm_head -> Identity (forward returns hidden states), soft-cap applied in ChunkedLossTrainer
stock = os.environ.get("VLMIN_STOCK_LOSS") == "1" # debug: compare against the stock full-logits loss
lm_weight = model.get_output_embeddings().weight
tcfg = model.config.get_text_config()
softcap = getattr(tcfg, "final_logit_softcapping", None)
if not stock:
tcfg.final_logit_softcapping = None
model.set_output_embeddings(nn.Identity())
lc = cfg["lora"]
model = get_peft_model(model, LoraConfig(
r=lc["r"], lora_alpha=lc["alpha"], lora_dropout=lc["dropout"],
target_modules=lc["target_regex"], task_type="CAUSAL_LM",
))
model.print_trainable_parameters()
targs = TrainingArguments(
output_dir=str(run_dir),
num_train_epochs=t["num_train_epochs"], max_steps=args.max_steps,
per_device_train_batch_size=t["per_device_train_batch_size"],
per_device_eval_batch_size=t["per_device_eval_batch_size"],
gradient_accumulation_steps=t["gradient_accumulation_steps"],
learning_rate=t["learning_rate"], lr_scheduler_type=t["lr_scheduler_type"],
warmup_steps=t["warmup_steps"], weight_decay=t["weight_decay"], max_grad_norm=t["max_grad_norm"],
optim=t["optim"], bf16=True,
gradient_checkpointing=True, gradient_checkpointing_kwargs={"use_reentrant": False},
logging_steps=t["logging_steps"],
save_strategy="steps", save_steps=t["save_steps"], save_total_limit=t.get("save_total_limit"), # None = keep all
eval_strategy="steps", eval_steps=t["eval_steps"],
dataloader_num_workers=t["dataloader_num_workers"], dataloader_pin_memory=True,
ddp_find_unused_parameters=False,
remove_unused_columns=False, report_to="none", seed=t["seed"], data_seed=t["seed"],
)
stopper = GracefulStop()
trainer = (Trainer if stock else ChunkedLossTrainer)(
model=model, args=targs, train_dataset=train_ds, eval_dataset=val_ds, data_collator=collator,
callbacks=[stopper, GpuMemLog()],
**({} if stock else dict(lm_weight=lm_weight, softcap=softcap)),
)
trainer.train(resume_from_checkpoint=last)
if stopper.requested: # stopped by signal: a checkpoint was saved; exit non-zero so a supervisor resumes instead of finishing
print("[train] stopped early by signal; checkpoint saved. Re-run to resume.", flush=True)
sys.exit(143)
final = run_dir / "final"
trainer.save_model(str(final))
processor.save_pretrained(str(final))
print(f"[train] done. adapter saved to {final}", flush=True)
if __name__ == "__main__":
main()