Instructions to use srivassid/drishti with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use srivassid/drishti with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("google/gemma-4-12B-it") model = PeftModel.from_pretrained(base_model, "srivassid/drishti") - Notebooks
- Google Colab
- Kaggle
Download code/train_data.py from srivassid/drishti: direct link, hf CLI and curl.
- Browser
- Download file 5.62 kB
-
https://huggingface.co/srivassid/drishti/resolve/main/code/train_data.py
- Command line
-
hf download hf://srivassid/drishti/code/train_data.py
-
curl -L -o train_data.py https://huggingface.co/srivassid/drishti/resolve/main/code/train_data.py
5.62 kB
| """Dataset + collator for Gemma 3 chat-format jsonl (the output of vlmin-mix). | |
| * JsonlVLMDataset keeps only byte offsets in memory and parses lines lazily, so 150k+ samples cost | |
| little RAM and `skip_first_batches` on resume never loads images. | |
| * Samples whose token length would exceed max_seq_len are dropped up front (truncating would cut | |
| the end-of-turn token or the image tokens). | |
| * Collator masks everything except the assistant reply (+ its <end_of_turn>), so loss is only on | |
| what the model should produce. | |
| """ | |
| import json | |
| import random | |
| import torch | |
| from PIL import Image | |
| Image.MAX_IMAGE_PIXELS = None # trusted local data; BSTD has a few 100+ MP photos | |
| IMAGE_TOKENS = 280 # upper bound: Gemma 4 max_soft_tokens (Gemma 3 used a fixed 256) | |
| TEMPLATE_OVERHEAD = 24 # <bos>, turn markers, newlines | |
| def load_image(path, max_side): | |
| """Decode cheaply (JPEG draft mode), RGB, and cap the long side; the processor does the final 896 resize.""" | |
| im = Image.open(path) | |
| im.draft("RGB", (max_side, max_side)) | |
| im = im.convert("RGB") | |
| im.thumbnail((max_side, max_side), Image.LANCZOS) | |
| return im | |
| class JsonlVLMDataset(torch.utils.data.Dataset): | |
| def __init__(self, path, tokenizer, max_seq_len, max_image_side=1344, limit=None, seed=0, longest=None): | |
| self.path, self.max_image_side = path, max_image_side | |
| offsets, lengths = [], [] | |
| with open(path, "rb") as f: | |
| pos = 0 | |
| for line in f: | |
| if line.strip(): | |
| offsets.append(pos) | |
| pos += len(line) | |
| if limit and len(offsets) > limit: | |
| offsets = sorted(random.Random(seed).sample(offsets, limit)) | |
| # token length filter (text tokens + image tokens), in batches | |
| keep, kept_len, dropped = [], [], 0 | |
| with open(path, "rb") as f: | |
| for i in range(0, len(offsets), 2000): | |
| chunk = offsets[i : i + 2000] | |
| texts, n_imgs = [], [] | |
| for off in chunk: | |
| f.seek(off) | |
| rec = json.loads(f.readline()) | |
| texts.append(" ".join(b["text"] for m in rec["messages"] for b in m["content"] if b["type"] == "text")) | |
| n_imgs.append(len(rec["images"])) | |
| ids = tokenizer(texts, add_special_tokens=False)["input_ids"] | |
| for off, t, n in zip(chunk, ids, n_imgs): | |
| total = len(t) + IMAGE_TOKENS * n + TEMPLATE_OVERHEAD | |
| if total <= max_seq_len: | |
| keep.append(off) | |
| kept_len.append(total) | |
| else: | |
| dropped += 1 | |
| if longest: # memory probe: only the K longest samples | |
| top = sorted(range(len(keep)), key=lambda i: -kept_len[i])[:longest] | |
| keep, kept_len = [keep[i] for i in top], [kept_len[i] for i in top] | |
| self.offsets, self.lengths, self.dropped = keep, kept_len, dropped | |
| self.bad_images = 0 | |
| def __len__(self): | |
| return len(self.offsets) | |
| def _build(self, off): | |
| with open(self.path, "rb") as f: | |
| f.seek(off) | |
| rec = json.loads(f.readline()) | |
| msgs = [] | |
| for m in rec["messages"]: | |
| content = [] | |
| for b in m["content"]: | |
| if b["type"] == "image": | |
| content.append({"type": "image", "image": load_image(b["image"], self.max_image_side)}) | |
| else: | |
| content.append(b) | |
| msgs.append({"role": m["role"], "content": content}) | |
| return {"messages": msgs, "id": rec["id"], "source": rec["source"], "task": rec["task"]} | |
| def __getitem__(self, i): | |
| # A corrupt image must not kill a multi-day run: fall through to the next sample. | |
| for k in range(10): | |
| try: | |
| return self._build(self.offsets[(i + k) % len(self.offsets)]) | |
| except Exception as e: # noqa: BLE001 | |
| self.bad_images += 1 | |
| print(f"[data] skipping sample {(i + k) % len(self.offsets)}: {type(e).__name__}: {e}", flush=True) | |
| raise RuntimeError("10 consecutive unreadable samples") | |
| class Collator: | |
| def __init__(self, processor, sot="<start_of_turn>", eot="<end_of_turn>"): | |
| # Gemma 3: <start_of_turn>/<end_of_turn>; Gemma 4: <|turn>/<turn|> | |
| self.processor = processor | |
| tok = processor.tokenizer | |
| tok.padding_side = "right" | |
| self.sot = tok.convert_tokens_to_ids(sot) | |
| self.eot = tok.convert_tokens_to_ids(eot) | |
| assert self.sot != tok.unk_token_id and self.eot != tok.unk_token_id, f"unknown turn tokens {sot} {eot}" | |
| # "<turn-start>model\n" -> the reply starts right after | |
| self.model_prefix_len = len(tok(f"{sot}model\n", add_special_tokens=False).input_ids) | |
| def __call__(self, batch): | |
| enc = self.processor.apply_chat_template( | |
| [b["messages"] for b in batch], | |
| tokenize=True, return_dict=True, return_tensors="pt", padding=True, | |
| ) | |
| ids = enc["input_ids"] | |
| labels = ids.clone() | |
| labels[enc["attention_mask"] == 0] = -100 | |
| for i in range(ids.shape[0]): | |
| row = ids[i] | |
| start = (row == self.sot).nonzero().flatten()[-1].item() + self.model_prefix_len | |
| end = (row == self.eot).nonzero().flatten()[-1].item() | |
| labels[i, :start] = -100 | |
| labels[i, end + 1 :] = -100 # trailing "\n" after <end_of_turn> | |
| enc["labels"] = labels | |
| enc["pixel_values"] = enc["pixel_values"].to(torch.bfloat16) | |
| return dict(enc) | |