Image-Text-to-Text
PEFT
Safetensors
gemma4
lora
qlora
ocr
indic
vision-language
File size: 5,624 Bytes
9df0eb7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
"""Dataset + collator for Gemma 3 chat-format jsonl (the output of vlmin-mix).

* JsonlVLMDataset keeps only byte offsets in memory and parses lines lazily, so 150k+ samples cost
  little RAM and `skip_first_batches` on resume never loads images.
* Samples whose token length would exceed max_seq_len are dropped up front (truncating would cut
  the end-of-turn token or the image tokens).
* Collator masks everything except the assistant reply (+ its <end_of_turn>), so loss is only on
  what the model should produce.
"""
import json
import random

import torch
from PIL import Image

Image.MAX_IMAGE_PIXELS = None  # trusted local data; BSTD has a few 100+ MP photos

IMAGE_TOKENS = 280  # upper bound: Gemma 4 max_soft_tokens (Gemma 3 used a fixed 256)
TEMPLATE_OVERHEAD = 24  # <bos>, turn markers, newlines


def load_image(path, max_side):
    """Decode cheaply (JPEG draft mode), RGB, and cap the long side; the processor does the final 896 resize."""
    im = Image.open(path)
    im.draft("RGB", (max_side, max_side))
    im = im.convert("RGB")
    im.thumbnail((max_side, max_side), Image.LANCZOS)
    return im


class JsonlVLMDataset(torch.utils.data.Dataset):
    def __init__(self, path, tokenizer, max_seq_len, max_image_side=1344, limit=None, seed=0, longest=None):
        self.path, self.max_image_side = path, max_image_side
        offsets, lengths = [], []
        with open(path, "rb") as f:
            pos = 0
            for line in f:
                if line.strip():
                    offsets.append(pos)
                pos += len(line)
        if limit and len(offsets) > limit:
            offsets = sorted(random.Random(seed).sample(offsets, limit))

        # token length filter (text tokens + image tokens), in batches
        keep, kept_len, dropped = [], [], 0
        with open(path, "rb") as f:
            for i in range(0, len(offsets), 2000):
                chunk = offsets[i : i + 2000]
                texts, n_imgs = [], []
                for off in chunk:
                    f.seek(off)
                    rec = json.loads(f.readline())
                    texts.append(" ".join(b["text"] for m in rec["messages"] for b in m["content"] if b["type"] == "text"))
                    n_imgs.append(len(rec["images"]))
                ids = tokenizer(texts, add_special_tokens=False)["input_ids"]
                for off, t, n in zip(chunk, ids, n_imgs):
                    total = len(t) + IMAGE_TOKENS * n + TEMPLATE_OVERHEAD
                    if total <= max_seq_len:
                        keep.append(off)
                        kept_len.append(total)
                    else:
                        dropped += 1
        if longest:  # memory probe: only the K longest samples
            top = sorted(range(len(keep)), key=lambda i: -kept_len[i])[:longest]
            keep, kept_len = [keep[i] for i in top], [kept_len[i] for i in top]
        self.offsets, self.lengths, self.dropped = keep, kept_len, dropped
        self.bad_images = 0

    def __len__(self):
        return len(self.offsets)

    def _build(self, off):
        with open(self.path, "rb") as f:
            f.seek(off)
            rec = json.loads(f.readline())
        msgs = []
        for m in rec["messages"]:
            content = []
            for b in m["content"]:
                if b["type"] == "image":
                    content.append({"type": "image", "image": load_image(b["image"], self.max_image_side)})
                else:
                    content.append(b)
            msgs.append({"role": m["role"], "content": content})
        return {"messages": msgs, "id": rec["id"], "source": rec["source"], "task": rec["task"]}

    def __getitem__(self, i):
        # A corrupt image must not kill a multi-day run: fall through to the next sample.
        for k in range(10):
            try:
                return self._build(self.offsets[(i + k) % len(self.offsets)])
            except Exception as e:  # noqa: BLE001
                self.bad_images += 1
                print(f"[data] skipping sample {(i + k) % len(self.offsets)}: {type(e).__name__}: {e}", flush=True)
        raise RuntimeError("10 consecutive unreadable samples")


class Collator:
    def __init__(self, processor, sot="<start_of_turn>", eot="<end_of_turn>"):
        # Gemma 3: <start_of_turn>/<end_of_turn>; Gemma 4: <|turn>/<turn|>
        self.processor = processor
        tok = processor.tokenizer
        tok.padding_side = "right"
        self.sot = tok.convert_tokens_to_ids(sot)
        self.eot = tok.convert_tokens_to_ids(eot)
        assert self.sot != tok.unk_token_id and self.eot != tok.unk_token_id, f"unknown turn tokens {sot} {eot}"
        # "<turn-start>model\n" -> the reply starts right after
        self.model_prefix_len = len(tok(f"{sot}model\n", add_special_tokens=False).input_ids)

    def __call__(self, batch):
        enc = self.processor.apply_chat_template(
            [b["messages"] for b in batch],
            tokenize=True, return_dict=True, return_tensors="pt", padding=True,
        )
        ids = enc["input_ids"]
        labels = ids.clone()
        labels[enc["attention_mask"] == 0] = -100
        for i in range(ids.shape[0]):
            row = ids[i]
            start = (row == self.sot).nonzero().flatten()[-1].item() + self.model_prefix_len
            end = (row == self.eot).nonzero().flatten()[-1].item()
            labels[i, :start] = -100
            labels[i, end + 1 :] = -100  # trailing "\n" after <end_of_turn>
        enc["labels"] = labels
        enc["pixel_values"] = enc["pixel_values"].to(torch.bfloat16)
        return dict(enc)