Instructions to use srivassid/drishti with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use srivassid/drishti with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("google/gemma-4-12B-it") model = PeftModel.from_pretrained(base_model, "srivassid/drishti") - Notebooks
- Google Colab
- Kaggle
Download code/train.py from srivassid/drishti: direct link, hf CLI and curl.
- Browser
- Download file 14.7 kB
-
https://huggingface.co/srivassid/drishti/resolve/main/code/train.py
- Command line
-
hf download hf://srivassid/drishti/code/train.py
-
curl -L -o train.py https://huggingface.co/srivassid/drishti/resolve/main/code/train.py
14.7 kB
| """vlmin-train: QLoRA fine-tuning of Gemma 3 12B on the mixed jsonl. | |
| vlmin-train --config configs/train.yaml # start, or resume if a checkpoint exists | |
| vlmin-train --check-data # no GPU: lengths, label masking, one decoded example | |
| vlmin-train --max-steps 6 --run-name smoke # smoke test into its own run dir | |
| Resumable: checkpoints (adapter + optimizer + scheduler + RNG + data position) are written to | |
| <output_root>/<run_name>/checkpoint-N on the Seagate drive; the newest `save_total_limit` are kept. Re-running the same | |
| command resumes from the newest one. SIGINT/SIGTERM stop cleanly: a checkpoint is saved first. | |
| A resume with a changed config (lr, batch, LoRA rank, dataset...) is refused unless --force-resume. | |
| """ | |
| import argparse | |
| import json | |
| import os | |
| import signal | |
| import sys | |
| from pathlib import Path | |
| os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") | |
| os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID") # stable GPU indices for max_memory | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") # less fragmentation OOM | |
| import torch # noqa: E402 | |
| import torch.nn as nn # noqa: E402 | |
| import torch.nn.functional as F # noqa: E402 | |
| import yaml # noqa: E402 | |
| from torch.utils.checkpoint import checkpoint # noqa: E402 | |
| from transformers import ( # noqa: E402 | |
| AutoModelForImageTextToText, AutoProcessor, BitsAndBytesConfig, Trainer, TrainerCallback, | |
| TrainingArguments, | |
| ) | |
| from transformers.trainer_utils import get_last_checkpoint # noqa: E402 | |
| from .train_data import Collator, JsonlVLMDataset # noqa: E402 | |
| # fields that must not change between a run and its resume | |
| IDENTITY_KEYS = [ | |
| ("model_id",), ("lora", "r"), ("lora", "alpha"), ("lora", "target_regex"), | |
| ("train", "learning_rate"), ("train", "num_train_epochs"), | |
| ("train", "warmup_steps"), ("train", "lr_scheduler_type"), ("data", "max_seq_len"), ("data", "train"), | |
| ] | |
| def _get(cfg, path): | |
| for k in path: | |
| cfg = cfg[k] | |
| return cfg | |
| class GracefulStop(TrainerCallback): | |
| """First SIGINT/SIGTERM: finish the current step, save a checkpoint, exit. Second one: die.""" | |
| def __init__(self): | |
| self.requested = False | |
| for sig in (signal.SIGINT, signal.SIGTERM): | |
| signal.signal(sig, self._handler) | |
| def _handler(self, signum, frame): | |
| if self.requested: | |
| sys.exit(130) | |
| self.requested = True | |
| print(f"\n[train] signal {signum}: saving a checkpoint after this step, then stopping " | |
| "(send again to kill immediately)", flush=True) | |
| def on_step_end(self, args, state, control, **kw): | |
| if self.requested: | |
| control.should_save = True | |
| control.should_training_stop = True | |
| class GpuMemLog(TrainerCallback): | |
| def on_log(self, args, state, control, logs=None, **kw): | |
| if logs is not None and torch.cuda.is_available(): | |
| logs["gpu_peak_gb"] = round(torch.cuda.max_memory_allocated() / 2**30, 2) | |
| def check_run_identity(run_dir, cfg, force, has_checkpoint): | |
| """Write the identity file on first start; on resume refuse if anything that shapes training changed. | |
| Per-device batch / grad-accum may change freely as long as their product (samples per step) doesn't.""" | |
| p = run_dir / "run_identity.json" | |
| ident = {".".join(k): _get(cfg, k) for k in IDENTITY_KEYS} | |
| ident["train.effective_batch"] = cfg["train"]["per_device_train_batch_size"] * cfg["train"]["gradient_accumulation_steps"] | |
| if p.exists() and has_checkpoint: | |
| old = json.loads(p.read_text()) | |
| diff = {k: (old.get(k), v) for k, v in ident.items() if old.get(k) != v} | |
| if diff and not force: | |
| sys.exit("Refusing to resume: config changed since this run started (old, new):\n" | |
| + json.dumps(diff, indent=2) + "\nUse a new run_name, or --force-resume if you are sure.") | |
| else: # fresh run, or an earlier attempt that never reached a checkpoint | |
| p.write_text(json.dumps(ident, indent=2)) | |
| def check_data(train_ds, collator, tokenizer, n=8): | |
| from collections import Counter | |
| batch = collator([train_ds[i] for i in range(n)]) | |
| ids, labels = batch["input_ids"], batch["labels"] | |
| print(f"train samples kept: {len(train_ds)} (dropped as too long: {train_ds.dropped})") | |
| print("batch shapes:", {k: tuple(v.shape) for k, v in batch.items()}) | |
| print("tasks in batch:", Counter(train_ds[i]["task"] for i in range(n))) | |
| print("bos count per row:", [(r == tokenizer.bos_token_id).sum().item() for r in ids]) | |
| frac = [(l != -100).sum().item() / (m.sum().item()) for l, m in zip(labels, batch["attention_mask"])] | |
| print("fraction of tokens with loss per row:", [round(f, 3) for f in frac]) | |
| i = 0 | |
| tgt = labels[i][labels[i] != -100] | |
| print("\nTRAINED-ON text (row 0):", repr(tokenizer.decode(tgt))[:300]) | |
| print("last 6 target tokens:", tokenizer.convert_ids_to_tokens(tgt[-6:].tolist())) | |
| unmasked_img = ((labels == train_ds_image_token(tokenizer)) & (labels != -100)).sum().item() | |
| print("image tokens inside loss region (must be 0):", unmasked_img) | |
| class ChunkedLossTrainer(Trainer): | |
| """Cross-entropy without ever materialising the full (tokens x 262k vocab) logits. | |
| The model's lm_head is replaced by nn.Identity, so the forward returns hidden states; the vocab projection, | |
| logit soft-capping and loss run here in `chunk`-token slices, each recomputed in backward (checkpoint). | |
| Only answer positions are projected. Saves ~3-4 GB per sample at 2048 tokens, which is what lets 10-12 GB | |
| cards train. Same maths as the stock loss (mean over answer tokens per micro-batch).""" | |
| def __init__(self, *a, lm_weight=None, softcap=None, chunk=256, **kw): | |
| super().__init__(*a, **kw) | |
| self.lm_weight, self.softcap, self.chunk = lm_weight, softcap, chunk | |
| self.model_accepts_loss_kwargs = False # we return a per-micro-batch mean; Trainer divides by grad-accum | |
| def _piece(self, h, y): | |
| lg = (h.to(self.lm_weight.dtype) @ self.lm_weight.t()).float() | |
| if self.softcap: | |
| lg = torch.tanh(lg / self.softcap) * self.softcap | |
| return F.cross_entropy(lg, y, reduction="sum") | |
| def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None): | |
| labels = inputs.pop("labels") | |
| hidden = model(**inputs).logits # = final hidden states (lm_head is Identity) | |
| h, y = hidden[:, :-1], labels[:, 1:].to(hidden.device) | |
| keep = y != -100 | |
| n = int(keep.sum()) | |
| if n == 0: | |
| return hidden.sum() * 0 | |
| h, y = h[keep], y[keep] | |
| total = 0.0 | |
| for i in range(0, n, self.chunk): | |
| total = total + checkpoint(self._piece, h[i:i + self.chunk], y[i:i + self.chunk], use_reentrant=False) | |
| loss = total / n | |
| return (loss, None) if return_outputs else loss | |
| def prediction_step(self, model, inputs, prediction_loss_only, ignore_keys=None): | |
| with torch.no_grad(), self.compute_loss_context_manager(): | |
| loss = self.compute_loss(model, dict(inputs)).detach() | |
| return loss, None, None | |
| IMAGE_TOKEN = "<image_soft_token>" # overridden from cfg["tokens"]["image"] | |
| def train_ds_image_token(tokenizer): | |
| return tokenizer.convert_tokens_to_ids(IMAGE_TOKEN) | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--config", default="configs/train.yaml") | |
| ap.add_argument("--run-name") | |
| ap.add_argument("--output-root") | |
| ap.add_argument("--max-steps", type=int, default=-1, help="stop after N optimizer steps (smoke tests)") | |
| ap.add_argument("--train-limit", type=int, help="use a random subset of N training samples") | |
| ap.add_argument("--longest", type=int, help="memory probe: train on only the K longest samples") | |
| ap.add_argument("--no-resume", action="store_true", help="ignore existing checkpoints (the run dir must be empty)") | |
| ap.add_argument("--force-resume", action="store_true", help="resume even if the config changed") | |
| ap.add_argument("--set", action="append", default=[], metavar="KEY=VAL", | |
| help="override a config value, e.g. --set train.save_steps=2 (repeatable)") | |
| ap.add_argument("--check-data", action="store_true", help="validate data pipeline on CPU and exit") | |
| args = ap.parse_args() | |
| cfg = yaml.safe_load(Path(args.config).read_text()) | |
| for kv in args.set: | |
| key, val = kv.split("=", 1) | |
| node = cfg | |
| *parents, leaf = key.split(".") | |
| for k in parents: | |
| node = node[k] | |
| val = yaml.safe_load(val) | |
| if isinstance(val, str): # YAML 1.1 reads "5e-5" as a string | |
| try: | |
| val = float(val) | |
| except ValueError: | |
| pass | |
| node[leaf] = val | |
| if args.run_name: | |
| cfg["run_name"] = args.run_name | |
| if args.output_root: | |
| cfg["output_root"] = args.output_root | |
| mount = cfg.get("require_mount") | |
| if mount and not os.path.ismount(mount): | |
| sys.exit(f"{mount} is not mounted (did the machine reboot?). Mount it and re-run; nothing was written.") | |
| run_dir = Path(cfg["output_root"]) / cfg["run_name"] | |
| d, t = cfg["data"], cfg["train"] | |
| processor = AutoProcessor.from_pretrained(cfg["model_id"]) | |
| tok = processor.tokenizer | |
| print("[train] indexing + length-filtering datasets ...", flush=True) | |
| train_ds = JsonlVLMDataset(d["train"], tok, d["max_seq_len"], d["max_image_side"], limit=args.train_limit, seed=t["seed"], longest=args.longest) | |
| val_ds = JsonlVLMDataset(d["val"], tok, d["max_seq_len"], d["max_image_side"], limit=d["eval_samples"], seed=t["seed"]) | |
| print(f"[train] train={len(train_ds)} (dropped {train_ds.dropped} too long) val={len(val_ds)}", flush=True) | |
| tk = cfg.get("tokens", {}) | |
| global IMAGE_TOKEN | |
| IMAGE_TOKEN = tk.get("image", IMAGE_TOKEN) | |
| collator = Collator(processor, tk.get("sot", "<start_of_turn>"), tk.get("eot", "<end_of_turn>")) | |
| if args.check_data: | |
| check_data(train_ds, collator, tok) | |
| return | |
| run_dir.mkdir(parents=True, exist_ok=True) | |
| last = None if args.no_resume else get_last_checkpoint(str(run_dir)) | |
| if args.no_resume and any(run_dir.glob("checkpoint-*")): | |
| sys.exit(f"--no-resume but {run_dir} already has checkpoints; pick a new --run-name") | |
| cfg_ident = cfg | |
| if args.train_limit or args.longest: | |
| cfg_ident = {**cfg, "data": {**d, "train": f"{d['train']}#limit{args.train_limit}#longest{args.longest}"}} | |
| check_run_identity(run_dir, cfg_ident, args.force_resume, last is not None) | |
| print(f"[train] run dir: {run_dir}\n[train] {'resuming from ' + last if last else 'starting fresh'}", flush=True) | |
| from peft import LoraConfig, get_peft_model | |
| bnb = BitsAndBytesConfig( | |
| load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, | |
| bnb_4bit_compute_dtype=torch.bfloat16, | |
| llm_int8_skip_modules=["vision_tower", "multi_modal_projector", "embed_vision", "embed_audio", "vision_embedder", "lm_head"], | |
| ) | |
| # device_map: {"": 0} = one GPU; "auto" + max_memory = layers split over several GPUs (naive pipeline) | |
| dm = cfg.get("device_map", {"": 0}) | |
| world = int(os.environ.get("WORLD_SIZE", "1")) | |
| if world > 1: # launched with torchrun: data parallel, one full model copy per GPU | |
| dm = {"": int(os.environ["LOCAL_RANK"])} | |
| kw = {"max_memory": {int(k): v for k, v in cfg["max_memory"].items()}} if cfg.get("max_memory") else {} | |
| model = AutoModelForImageTextToText.from_pretrained( | |
| cfg["model_id"], quantization_config=bnb, dtype=torch.bfloat16, | |
| attn_implementation=cfg["attn_implementation"], device_map=dm, **kw, | |
| ) | |
| print("[train] device map:", {k: v for k, v in list(getattr(model, "hf_device_map", {}).items())[:6]}, "...", flush=True) | |
| model.config.use_cache = False | |
| # chunked loss: lm_head -> Identity (forward returns hidden states), soft-cap applied in ChunkedLossTrainer | |
| stock = os.environ.get("VLMIN_STOCK_LOSS") == "1" # debug: compare against the stock full-logits loss | |
| lm_weight = model.get_output_embeddings().weight | |
| tcfg = model.config.get_text_config() | |
| softcap = getattr(tcfg, "final_logit_softcapping", None) | |
| if not stock: | |
| tcfg.final_logit_softcapping = None | |
| model.set_output_embeddings(nn.Identity()) | |
| lc = cfg["lora"] | |
| model = get_peft_model(model, LoraConfig( | |
| r=lc["r"], lora_alpha=lc["alpha"], lora_dropout=lc["dropout"], | |
| target_modules=lc["target_regex"], task_type="CAUSAL_LM", | |
| )) | |
| model.print_trainable_parameters() | |
| targs = TrainingArguments( | |
| output_dir=str(run_dir), | |
| num_train_epochs=t["num_train_epochs"], max_steps=args.max_steps, | |
| per_device_train_batch_size=t["per_device_train_batch_size"], | |
| per_device_eval_batch_size=t["per_device_eval_batch_size"], | |
| gradient_accumulation_steps=t["gradient_accumulation_steps"], | |
| learning_rate=t["learning_rate"], lr_scheduler_type=t["lr_scheduler_type"], | |
| warmup_steps=t["warmup_steps"], weight_decay=t["weight_decay"], max_grad_norm=t["max_grad_norm"], | |
| optim=t["optim"], bf16=True, | |
| gradient_checkpointing=True, gradient_checkpointing_kwargs={"use_reentrant": False}, | |
| logging_steps=t["logging_steps"], | |
| save_strategy="steps", save_steps=t["save_steps"], save_total_limit=t.get("save_total_limit"), # None = keep all | |
| eval_strategy="steps", eval_steps=t["eval_steps"], | |
| dataloader_num_workers=t["dataloader_num_workers"], dataloader_pin_memory=True, | |
| ddp_find_unused_parameters=False, | |
| remove_unused_columns=False, report_to="none", seed=t["seed"], data_seed=t["seed"], | |
| ) | |
| stopper = GracefulStop() | |
| trainer = (Trainer if stock else ChunkedLossTrainer)( | |
| model=model, args=targs, train_dataset=train_ds, eval_dataset=val_ds, data_collator=collator, | |
| callbacks=[stopper, GpuMemLog()], | |
| **({} if stock else dict(lm_weight=lm_weight, softcap=softcap)), | |
| ) | |
| trainer.train(resume_from_checkpoint=last) | |
| if stopper.requested: # stopped by signal: a checkpoint was saved; exit non-zero so a supervisor resumes instead of finishing | |
| print("[train] stopped early by signal; checkpoint saved. Re-run to resume.", flush=True) | |
| sys.exit(143) | |
| final = run_dir / "final" | |
| trainer.save_model(str(final)) | |
| processor.save_pretrained(str(final)) | |
| print(f"[train] done. adapter saved to {final}", flush=True) | |
| if __name__ == "__main__": | |
| main() | |