Instructions to use srivassid/drishti with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use srivassid/drishti with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("google/gemma-4-12B-it") model = PeftModel.from_pretrained(base_model, "srivassid/drishti") - Notebooks
- Google Colab
- Kaggle
File size: 5,624 Bytes
9df0eb7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | """Dataset + collator for Gemma 3 chat-format jsonl (the output of vlmin-mix).
* JsonlVLMDataset keeps only byte offsets in memory and parses lines lazily, so 150k+ samples cost
little RAM and `skip_first_batches` on resume never loads images.
* Samples whose token length would exceed max_seq_len are dropped up front (truncating would cut
the end-of-turn token or the image tokens).
* Collator masks everything except the assistant reply (+ its <end_of_turn>), so loss is only on
what the model should produce.
"""
import json
import random
import torch
from PIL import Image
Image.MAX_IMAGE_PIXELS = None # trusted local data; BSTD has a few 100+ MP photos
IMAGE_TOKENS = 280 # upper bound: Gemma 4 max_soft_tokens (Gemma 3 used a fixed 256)
TEMPLATE_OVERHEAD = 24 # <bos>, turn markers, newlines
def load_image(path, max_side):
"""Decode cheaply (JPEG draft mode), RGB, and cap the long side; the processor does the final 896 resize."""
im = Image.open(path)
im.draft("RGB", (max_side, max_side))
im = im.convert("RGB")
im.thumbnail((max_side, max_side), Image.LANCZOS)
return im
class JsonlVLMDataset(torch.utils.data.Dataset):
def __init__(self, path, tokenizer, max_seq_len, max_image_side=1344, limit=None, seed=0, longest=None):
self.path, self.max_image_side = path, max_image_side
offsets, lengths = [], []
with open(path, "rb") as f:
pos = 0
for line in f:
if line.strip():
offsets.append(pos)
pos += len(line)
if limit and len(offsets) > limit:
offsets = sorted(random.Random(seed).sample(offsets, limit))
# token length filter (text tokens + image tokens), in batches
keep, kept_len, dropped = [], [], 0
with open(path, "rb") as f:
for i in range(0, len(offsets), 2000):
chunk = offsets[i : i + 2000]
texts, n_imgs = [], []
for off in chunk:
f.seek(off)
rec = json.loads(f.readline())
texts.append(" ".join(b["text"] for m in rec["messages"] for b in m["content"] if b["type"] == "text"))
n_imgs.append(len(rec["images"]))
ids = tokenizer(texts, add_special_tokens=False)["input_ids"]
for off, t, n in zip(chunk, ids, n_imgs):
total = len(t) + IMAGE_TOKENS * n + TEMPLATE_OVERHEAD
if total <= max_seq_len:
keep.append(off)
kept_len.append(total)
else:
dropped += 1
if longest: # memory probe: only the K longest samples
top = sorted(range(len(keep)), key=lambda i: -kept_len[i])[:longest]
keep, kept_len = [keep[i] for i in top], [kept_len[i] for i in top]
self.offsets, self.lengths, self.dropped = keep, kept_len, dropped
self.bad_images = 0
def __len__(self):
return len(self.offsets)
def _build(self, off):
with open(self.path, "rb") as f:
f.seek(off)
rec = json.loads(f.readline())
msgs = []
for m in rec["messages"]:
content = []
for b in m["content"]:
if b["type"] == "image":
content.append({"type": "image", "image": load_image(b["image"], self.max_image_side)})
else:
content.append(b)
msgs.append({"role": m["role"], "content": content})
return {"messages": msgs, "id": rec["id"], "source": rec["source"], "task": rec["task"]}
def __getitem__(self, i):
# A corrupt image must not kill a multi-day run: fall through to the next sample.
for k in range(10):
try:
return self._build(self.offsets[(i + k) % len(self.offsets)])
except Exception as e: # noqa: BLE001
self.bad_images += 1
print(f"[data] skipping sample {(i + k) % len(self.offsets)}: {type(e).__name__}: {e}", flush=True)
raise RuntimeError("10 consecutive unreadable samples")
class Collator:
def __init__(self, processor, sot="<start_of_turn>", eot="<end_of_turn>"):
# Gemma 3: <start_of_turn>/<end_of_turn>; Gemma 4: <|turn>/<turn|>
self.processor = processor
tok = processor.tokenizer
tok.padding_side = "right"
self.sot = tok.convert_tokens_to_ids(sot)
self.eot = tok.convert_tokens_to_ids(eot)
assert self.sot != tok.unk_token_id and self.eot != tok.unk_token_id, f"unknown turn tokens {sot} {eot}"
# "<turn-start>model\n" -> the reply starts right after
self.model_prefix_len = len(tok(f"{sot}model\n", add_special_tokens=False).input_ids)
def __call__(self, batch):
enc = self.processor.apply_chat_template(
[b["messages"] for b in batch],
tokenize=True, return_dict=True, return_tensors="pt", padding=True,
)
ids = enc["input_ids"]
labels = ids.clone()
labels[enc["attention_mask"] == 0] = -100
for i in range(ids.shape[0]):
row = ids[i]
start = (row == self.sot).nonzero().flatten()[-1].item() + self.model_prefix_len
end = (row == self.eot).nonzero().flatten()[-1].item()
labels[i, :start] = -100
labels[i, end + 1 :] = -100 # trailing "\n" after <end_of_turn>
enc["labels"] = labels
enc["pixel_values"] = enc["pixel_values"].to(torch.bfloat16)
return dict(enc)
|