"""Dataset + collator for Gemma 3 chat-format jsonl (the output of vlmin-mix). * JsonlVLMDataset keeps only byte offsets in memory and parses lines lazily, so 150k+ samples cost little RAM and `skip_first_batches` on resume never loads images. * Samples whose token length would exceed max_seq_len are dropped up front (truncating would cut the end-of-turn token or the image tokens). * Collator masks everything except the assistant reply (+ its ), so loss is only on what the model should produce. """ import json import random import torch from PIL import Image Image.MAX_IMAGE_PIXELS = None # trusted local data; BSTD has a few 100+ MP photos IMAGE_TOKENS = 280 # upper bound: Gemma 4 max_soft_tokens (Gemma 3 used a fixed 256) TEMPLATE_OVERHEAD = 24 # , turn markers, newlines def load_image(path, max_side): """Decode cheaply (JPEG draft mode), RGB, and cap the long side; the processor does the final 896 resize.""" im = Image.open(path) im.draft("RGB", (max_side, max_side)) im = im.convert("RGB") im.thumbnail((max_side, max_side), Image.LANCZOS) return im class JsonlVLMDataset(torch.utils.data.Dataset): def __init__(self, path, tokenizer, max_seq_len, max_image_side=1344, limit=None, seed=0, longest=None): self.path, self.max_image_side = path, max_image_side offsets, lengths = [], [] with open(path, "rb") as f: pos = 0 for line in f: if line.strip(): offsets.append(pos) pos += len(line) if limit and len(offsets) > limit: offsets = sorted(random.Random(seed).sample(offsets, limit)) # token length filter (text tokens + image tokens), in batches keep, kept_len, dropped = [], [], 0 with open(path, "rb") as f: for i in range(0, len(offsets), 2000): chunk = offsets[i : i + 2000] texts, n_imgs = [], [] for off in chunk: f.seek(off) rec = json.loads(f.readline()) texts.append(" ".join(b["text"] for m in rec["messages"] for b in m["content"] if b["type"] == "text")) n_imgs.append(len(rec["images"])) ids = tokenizer(texts, add_special_tokens=False)["input_ids"] for off, t, n in zip(chunk, ids, n_imgs): total = len(t) + IMAGE_TOKENS * n + TEMPLATE_OVERHEAD if total <= max_seq_len: keep.append(off) kept_len.append(total) else: dropped += 1 if longest: # memory probe: only the K longest samples top = sorted(range(len(keep)), key=lambda i: -kept_len[i])[:longest] keep, kept_len = [keep[i] for i in top], [kept_len[i] for i in top] self.offsets, self.lengths, self.dropped = keep, kept_len, dropped self.bad_images = 0 def __len__(self): return len(self.offsets) def _build(self, off): with open(self.path, "rb") as f: f.seek(off) rec = json.loads(f.readline()) msgs = [] for m in rec["messages"]: content = [] for b in m["content"]: if b["type"] == "image": content.append({"type": "image", "image": load_image(b["image"], self.max_image_side)}) else: content.append(b) msgs.append({"role": m["role"], "content": content}) return {"messages": msgs, "id": rec["id"], "source": rec["source"], "task": rec["task"]} def __getitem__(self, i): # A corrupt image must not kill a multi-day run: fall through to the next sample. for k in range(10): try: return self._build(self.offsets[(i + k) % len(self.offsets)]) except Exception as e: # noqa: BLE001 self.bad_images += 1 print(f"[data] skipping sample {(i + k) % len(self.offsets)}: {type(e).__name__}: {e}", flush=True) raise RuntimeError("10 consecutive unreadable samples") class Collator: def __init__(self, processor, sot="", eot=""): # Gemma 3: /; Gemma 4: <|turn>/ self.processor = processor tok = processor.tokenizer tok.padding_side = "right" self.sot = tok.convert_tokens_to_ids(sot) self.eot = tok.convert_tokens_to_ids(eot) assert self.sot != tok.unk_token_id and self.eot != tok.unk_token_id, f"unknown turn tokens {sot} {eot}" # "model\n" -> the reply starts right after self.model_prefix_len = len(tok(f"{sot}model\n", add_special_tokens=False).input_ids) def __call__(self, batch): enc = self.processor.apply_chat_template( [b["messages"] for b in batch], tokenize=True, return_dict=True, return_tensors="pt", padding=True, ) ids = enc["input_ids"] labels = ids.clone() labels[enc["attention_mask"] == 0] = -100 for i in range(ids.shape[0]): row = ids[i] start = (row == self.sot).nonzero().flatten()[-1].item() + self.model_prefix_len end = (row == self.eot).nonzero().flatten()[-1].item() labels[i, :start] = -100 labels[i, end + 1 :] = -100 # trailing "\n" after enc["labels"] = labels enc["pixel_values"] = enc["pixel_values"].to(torch.bfloat16) return dict(enc)