Instructions to use srivassid/drishti with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use srivassid/drishti with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("google/gemma-4-12B-it") model = PeftModel.from_pretrained(base_model, "srivassid/drishti") - Notebooks
- Google Colab
- Kaggle
File size: 14,665 Bytes
9df0eb7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 | """vlmin-train: QLoRA fine-tuning of Gemma 3 12B on the mixed jsonl.
vlmin-train --config configs/train.yaml # start, or resume if a checkpoint exists
vlmin-train --check-data # no GPU: lengths, label masking, one decoded example
vlmin-train --max-steps 6 --run-name smoke # smoke test into its own run dir
Resumable: checkpoints (adapter + optimizer + scheduler + RNG + data position) are written to
<output_root>/<run_name>/checkpoint-N on the Seagate drive; the newest `save_total_limit` are kept. Re-running the same
command resumes from the newest one. SIGINT/SIGTERM stop cleanly: a checkpoint is saved first.
A resume with a changed config (lr, batch, LoRA rank, dataset...) is refused unless --force-resume.
"""
import argparse
import json
import os
import signal
import sys
from pathlib import Path
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID") # stable GPU indices for max_memory
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") # less fragmentation OOM
import torch # noqa: E402
import torch.nn as nn # noqa: E402
import torch.nn.functional as F # noqa: E402
import yaml # noqa: E402
from torch.utils.checkpoint import checkpoint # noqa: E402
from transformers import ( # noqa: E402
AutoModelForImageTextToText, AutoProcessor, BitsAndBytesConfig, Trainer, TrainerCallback,
TrainingArguments,
)
from transformers.trainer_utils import get_last_checkpoint # noqa: E402
from .train_data import Collator, JsonlVLMDataset # noqa: E402
# fields that must not change between a run and its resume
IDENTITY_KEYS = [
("model_id",), ("lora", "r"), ("lora", "alpha"), ("lora", "target_regex"),
("train", "learning_rate"), ("train", "num_train_epochs"),
("train", "warmup_steps"), ("train", "lr_scheduler_type"), ("data", "max_seq_len"), ("data", "train"),
]
def _get(cfg, path):
for k in path:
cfg = cfg[k]
return cfg
class GracefulStop(TrainerCallback):
"""First SIGINT/SIGTERM: finish the current step, save a checkpoint, exit. Second one: die."""
def __init__(self):
self.requested = False
for sig in (signal.SIGINT, signal.SIGTERM):
signal.signal(sig, self._handler)
def _handler(self, signum, frame):
if self.requested:
sys.exit(130)
self.requested = True
print(f"\n[train] signal {signum}: saving a checkpoint after this step, then stopping "
"(send again to kill immediately)", flush=True)
def on_step_end(self, args, state, control, **kw):
if self.requested:
control.should_save = True
control.should_training_stop = True
class GpuMemLog(TrainerCallback):
def on_log(self, args, state, control, logs=None, **kw):
if logs is not None and torch.cuda.is_available():
logs["gpu_peak_gb"] = round(torch.cuda.max_memory_allocated() / 2**30, 2)
def check_run_identity(run_dir, cfg, force, has_checkpoint):
"""Write the identity file on first start; on resume refuse if anything that shapes training changed.
Per-device batch / grad-accum may change freely as long as their product (samples per step) doesn't."""
p = run_dir / "run_identity.json"
ident = {".".join(k): _get(cfg, k) for k in IDENTITY_KEYS}
ident["train.effective_batch"] = cfg["train"]["per_device_train_batch_size"] * cfg["train"]["gradient_accumulation_steps"]
if p.exists() and has_checkpoint:
old = json.loads(p.read_text())
diff = {k: (old.get(k), v) for k, v in ident.items() if old.get(k) != v}
if diff and not force:
sys.exit("Refusing to resume: config changed since this run started (old, new):\n"
+ json.dumps(diff, indent=2) + "\nUse a new run_name, or --force-resume if you are sure.")
else: # fresh run, or an earlier attempt that never reached a checkpoint
p.write_text(json.dumps(ident, indent=2))
def check_data(train_ds, collator, tokenizer, n=8):
from collections import Counter
batch = collator([train_ds[i] for i in range(n)])
ids, labels = batch["input_ids"], batch["labels"]
print(f"train samples kept: {len(train_ds)} (dropped as too long: {train_ds.dropped})")
print("batch shapes:", {k: tuple(v.shape) for k, v in batch.items()})
print("tasks in batch:", Counter(train_ds[i]["task"] for i in range(n)))
print("bos count per row:", [(r == tokenizer.bos_token_id).sum().item() for r in ids])
frac = [(l != -100).sum().item() / (m.sum().item()) for l, m in zip(labels, batch["attention_mask"])]
print("fraction of tokens with loss per row:", [round(f, 3) for f in frac])
i = 0
tgt = labels[i][labels[i] != -100]
print("\nTRAINED-ON text (row 0):", repr(tokenizer.decode(tgt))[:300])
print("last 6 target tokens:", tokenizer.convert_ids_to_tokens(tgt[-6:].tolist()))
unmasked_img = ((labels == train_ds_image_token(tokenizer)) & (labels != -100)).sum().item()
print("image tokens inside loss region (must be 0):", unmasked_img)
class ChunkedLossTrainer(Trainer):
"""Cross-entropy without ever materialising the full (tokens x 262k vocab) logits.
The model's lm_head is replaced by nn.Identity, so the forward returns hidden states; the vocab projection,
logit soft-capping and loss run here in `chunk`-token slices, each recomputed in backward (checkpoint).
Only answer positions are projected. Saves ~3-4 GB per sample at 2048 tokens, which is what lets 10-12 GB
cards train. Same maths as the stock loss (mean over answer tokens per micro-batch)."""
def __init__(self, *a, lm_weight=None, softcap=None, chunk=256, **kw):
super().__init__(*a, **kw)
self.lm_weight, self.softcap, self.chunk = lm_weight, softcap, chunk
self.model_accepts_loss_kwargs = False # we return a per-micro-batch mean; Trainer divides by grad-accum
def _piece(self, h, y):
lg = (h.to(self.lm_weight.dtype) @ self.lm_weight.t()).float()
if self.softcap:
lg = torch.tanh(lg / self.softcap) * self.softcap
return F.cross_entropy(lg, y, reduction="sum")
def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
labels = inputs.pop("labels")
hidden = model(**inputs).logits # = final hidden states (lm_head is Identity)
h, y = hidden[:, :-1], labels[:, 1:].to(hidden.device)
keep = y != -100
n = int(keep.sum())
if n == 0:
return hidden.sum() * 0
h, y = h[keep], y[keep]
total = 0.0
for i in range(0, n, self.chunk):
total = total + checkpoint(self._piece, h[i:i + self.chunk], y[i:i + self.chunk], use_reentrant=False)
loss = total / n
return (loss, None) if return_outputs else loss
def prediction_step(self, model, inputs, prediction_loss_only, ignore_keys=None):
with torch.no_grad(), self.compute_loss_context_manager():
loss = self.compute_loss(model, dict(inputs)).detach()
return loss, None, None
IMAGE_TOKEN = "<image_soft_token>" # overridden from cfg["tokens"]["image"]
def train_ds_image_token(tokenizer):
return tokenizer.convert_tokens_to_ids(IMAGE_TOKEN)
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--config", default="configs/train.yaml")
ap.add_argument("--run-name")
ap.add_argument("--output-root")
ap.add_argument("--max-steps", type=int, default=-1, help="stop after N optimizer steps (smoke tests)")
ap.add_argument("--train-limit", type=int, help="use a random subset of N training samples")
ap.add_argument("--longest", type=int, help="memory probe: train on only the K longest samples")
ap.add_argument("--no-resume", action="store_true", help="ignore existing checkpoints (the run dir must be empty)")
ap.add_argument("--force-resume", action="store_true", help="resume even if the config changed")
ap.add_argument("--set", action="append", default=[], metavar="KEY=VAL",
help="override a config value, e.g. --set train.save_steps=2 (repeatable)")
ap.add_argument("--check-data", action="store_true", help="validate data pipeline on CPU and exit")
args = ap.parse_args()
cfg = yaml.safe_load(Path(args.config).read_text())
for kv in args.set:
key, val = kv.split("=", 1)
node = cfg
*parents, leaf = key.split(".")
for k in parents:
node = node[k]
val = yaml.safe_load(val)
if isinstance(val, str): # YAML 1.1 reads "5e-5" as a string
try:
val = float(val)
except ValueError:
pass
node[leaf] = val
if args.run_name:
cfg["run_name"] = args.run_name
if args.output_root:
cfg["output_root"] = args.output_root
mount = cfg.get("require_mount")
if mount and not os.path.ismount(mount):
sys.exit(f"{mount} is not mounted (did the machine reboot?). Mount it and re-run; nothing was written.")
run_dir = Path(cfg["output_root"]) / cfg["run_name"]
d, t = cfg["data"], cfg["train"]
processor = AutoProcessor.from_pretrained(cfg["model_id"])
tok = processor.tokenizer
print("[train] indexing + length-filtering datasets ...", flush=True)
train_ds = JsonlVLMDataset(d["train"], tok, d["max_seq_len"], d["max_image_side"], limit=args.train_limit, seed=t["seed"], longest=args.longest)
val_ds = JsonlVLMDataset(d["val"], tok, d["max_seq_len"], d["max_image_side"], limit=d["eval_samples"], seed=t["seed"])
print(f"[train] train={len(train_ds)} (dropped {train_ds.dropped} too long) val={len(val_ds)}", flush=True)
tk = cfg.get("tokens", {})
global IMAGE_TOKEN
IMAGE_TOKEN = tk.get("image", IMAGE_TOKEN)
collator = Collator(processor, tk.get("sot", "<start_of_turn>"), tk.get("eot", "<end_of_turn>"))
if args.check_data:
check_data(train_ds, collator, tok)
return
run_dir.mkdir(parents=True, exist_ok=True)
last = None if args.no_resume else get_last_checkpoint(str(run_dir))
if args.no_resume and any(run_dir.glob("checkpoint-*")):
sys.exit(f"--no-resume but {run_dir} already has checkpoints; pick a new --run-name")
cfg_ident = cfg
if args.train_limit or args.longest:
cfg_ident = {**cfg, "data": {**d, "train": f"{d['train']}#limit{args.train_limit}#longest{args.longest}"}}
check_run_identity(run_dir, cfg_ident, args.force_resume, last is not None)
print(f"[train] run dir: {run_dir}\n[train] {'resuming from ' + last if last else 'starting fresh'}", flush=True)
from peft import LoraConfig, get_peft_model
bnb = BitsAndBytesConfig(
load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.bfloat16,
llm_int8_skip_modules=["vision_tower", "multi_modal_projector", "embed_vision", "embed_audio", "vision_embedder", "lm_head"],
)
# device_map: {"": 0} = one GPU; "auto" + max_memory = layers split over several GPUs (naive pipeline)
dm = cfg.get("device_map", {"": 0})
world = int(os.environ.get("WORLD_SIZE", "1"))
if world > 1: # launched with torchrun: data parallel, one full model copy per GPU
dm = {"": int(os.environ["LOCAL_RANK"])}
kw = {"max_memory": {int(k): v for k, v in cfg["max_memory"].items()}} if cfg.get("max_memory") else {}
model = AutoModelForImageTextToText.from_pretrained(
cfg["model_id"], quantization_config=bnb, dtype=torch.bfloat16,
attn_implementation=cfg["attn_implementation"], device_map=dm, **kw,
)
print("[train] device map:", {k: v for k, v in list(getattr(model, "hf_device_map", {}).items())[:6]}, "...", flush=True)
model.config.use_cache = False
# chunked loss: lm_head -> Identity (forward returns hidden states), soft-cap applied in ChunkedLossTrainer
stock = os.environ.get("VLMIN_STOCK_LOSS") == "1" # debug: compare against the stock full-logits loss
lm_weight = model.get_output_embeddings().weight
tcfg = model.config.get_text_config()
softcap = getattr(tcfg, "final_logit_softcapping", None)
if not stock:
tcfg.final_logit_softcapping = None
model.set_output_embeddings(nn.Identity())
lc = cfg["lora"]
model = get_peft_model(model, LoraConfig(
r=lc["r"], lora_alpha=lc["alpha"], lora_dropout=lc["dropout"],
target_modules=lc["target_regex"], task_type="CAUSAL_LM",
))
model.print_trainable_parameters()
targs = TrainingArguments(
output_dir=str(run_dir),
num_train_epochs=t["num_train_epochs"], max_steps=args.max_steps,
per_device_train_batch_size=t["per_device_train_batch_size"],
per_device_eval_batch_size=t["per_device_eval_batch_size"],
gradient_accumulation_steps=t["gradient_accumulation_steps"],
learning_rate=t["learning_rate"], lr_scheduler_type=t["lr_scheduler_type"],
warmup_steps=t["warmup_steps"], weight_decay=t["weight_decay"], max_grad_norm=t["max_grad_norm"],
optim=t["optim"], bf16=True,
gradient_checkpointing=True, gradient_checkpointing_kwargs={"use_reentrant": False},
logging_steps=t["logging_steps"],
save_strategy="steps", save_steps=t["save_steps"], save_total_limit=t.get("save_total_limit"), # None = keep all
eval_strategy="steps", eval_steps=t["eval_steps"],
dataloader_num_workers=t["dataloader_num_workers"], dataloader_pin_memory=True,
ddp_find_unused_parameters=False,
remove_unused_columns=False, report_to="none", seed=t["seed"], data_seed=t["seed"],
)
stopper = GracefulStop()
trainer = (Trainer if stock else ChunkedLossTrainer)(
model=model, args=targs, train_dataset=train_ds, eval_dataset=val_ds, data_collator=collator,
callbacks=[stopper, GpuMemLog()],
**({} if stock else dict(lm_weight=lm_weight, softcap=softcap)),
)
trainer.train(resume_from_checkpoint=last)
if stopper.requested: # stopped by signal: a checkpoint was saved; exit non-zero so a supervisor resumes instead of finishing
print("[train] stopped early by signal; checkpoint saved. Re-run to resume.", flush=True)
sys.exit(143)
final = run_dir / "final"
trainer.save_model(str(final))
processor.save_pretrained(str(final))
print(f"[train] done. adapter saved to {final}", flush=True)
if __name__ == "__main__":
main()
|