Image-Text-to-Text
PEFT
Safetensors
gemma4
lora
qlora
ocr
indic
vision-language
File size: 7,397 Bytes
9df0eb7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
"""Evaluate base vs fine-tuned Gemma on IndicVisionBench (a benchmark that is NOT in the training mix).

    CUDA_VISIBLE_DEVICES=1 python -m vlmin.eval_bench --bench /mnt/hdd/vlmin/bench/IndicVisionBench \
        --out base.json                       # base model
    CUDA_VISIBLE_DEVICES=2 python -m vlmin.eval_bench ... --adapter runs/<run>/final --out tuned.json

Tasks (objective metrics only, no LLM judge):
  ocr  page transcription, scored by CER on the first --ocr-chars characters (prediction and reference are
       both cut to that length, so a token limit doesn't penalise long pages).  Lower is better.
  mmt  translate the English caption of an image into one Indic language, scored by chrF.  Higher is better.
Prompts reuse the wording seen in training (prompts.OCR_PAGE[0]) so the comparison is not about prompt format.
"""
import argparse
import io
import json
import os
import random
import time
from collections import defaultdict
from pathlib import Path

os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID")  # same GPU numbering as nvidia-smi and train.py
os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")

import pandas as pd  # noqa: E402
import torch  # noqa: E402
from PIL import Image  # noqa: E402
from transformers import AutoModelForImageTextToText, AutoProcessor, BitsAndBytesConfig  # noqa: E402

from . import prompts  # noqa: E402

MMT_LANGS = ["Hindi", "Bengali", "Gujarati", "Kannada", "Malayalam", "Marathi", "Odia", "Punjabi", "Tamil", "Telugu"]


def load_model(model_id, adapter):
    bnb = BitsAndBytesConfig(
        load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True,
        bnb_4bit_compute_dtype=torch.bfloat16,
        llm_int8_skip_modules=["vision_tower", "multi_modal_projector", "embed_vision", "embed_audio",
                               "vision_embedder", "lm_head"],
    )
    model = AutoModelForImageTextToText.from_pretrained(
        model_id, quantization_config=bnb, dtype=torch.bfloat16, attn_implementation="sdpa", device_map={"": 0})
    if adapter:
        from peft import PeftModel
        model = PeftModel.from_pretrained(model, adapter)
    model.eval()
    return model


def to_image(cell, max_side=1344):
    im = Image.open(io.BytesIO(cell["bytes"])).convert("RGB")
    im.thumbnail((max_side, max_side), Image.LANCZOS)
    return im


@torch.no_grad()
def generate(model, processor, items, max_new_tokens, bs):
    """items: list of (PIL image, prompt text). Greedy, left-padded batches. Returns list of strings."""
    processor.tokenizer.padding_side = "left"
    outs = []
    for i in range(0, len(items), bs):
        chunk = items[i:i + bs]
        msgs = [[{"role": "user", "content": [{"type": "image", "image": im}, {"type": "text", "text": p}]}]
                for im, p in chunk]
        enc = processor.apply_chat_template(msgs, tokenize=True, return_dict=True, return_tensors="pt",
                                            padding=True, add_generation_prompt=True)
        enc = {k: v.to("cuda") for k, v in enc.items()}
        enc["pixel_values"] = enc["pixel_values"].to(torch.bfloat16)
        n_in = enc["input_ids"].shape[1]
        gen = model.generate(**enc, max_new_tokens=max_new_tokens, do_sample=False)
        outs += processor.batch_decode(gen[:, n_in:], skip_special_tokens=True)
        print(f"  generated {min(i + bs, len(items))}/{len(items)}", flush=True)
    return outs


def cer(pred, ref, n):
    from rapidfuzz.distance import Levenshtein
    ref, pred = ref[:n], pred[:n]
    return Levenshtein.distance(pred, ref) / max(1, len(ref))


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--bench", required=True)
    ap.add_argument("--model", default="/run/media/sid/disk/VLM-IN/models/gemma-4-12B-it")
    ap.add_argument("--adapter")
    ap.add_argument("--out", required=True)
    ap.add_argument("--tasks", default="ocr,mmt")
    ap.add_argument("--ocr-per-lang", type=int, default=20)
    ap.add_argument("--ocr-chars", type=int, default=600)
    ap.add_argument("--max-new-tokens", type=int, default=768)
    ap.add_argument("--bs", type=int, default=4)
    ap.add_argument("--seed", type=int, default=0)
    args = ap.parse_args()
    bench = Path(args.bench)

    t0 = time.time()
    processor = AutoProcessor.from_pretrained(args.model)
    model = load_model(args.model, args.adapter)
    print(f"[eval] model loaded in {time.time() - t0:.0f}s  adapter={args.adapter}", flush=True)
    res = {"adapter": args.adapter, "tasks": {}}

    if "ocr" in args.tasks:
        df = pd.concat([pd.read_parquet(p) for p in sorted((bench / "ocr").glob("*.parquet"))], ignore_index=True)
        rng = random.Random(args.seed)
        pick = []
        for lang, g in df.groupby("language"):
            idx = list(g.index)
            rng.shuffle(idx)
            pick += idx[:args.ocr_per_lang]
        sub = df.loc[sorted(pick)]
        print(f"[eval] ocr: {len(sub)} pages, languages={sorted(sub['language'].unique())}", flush=True)
        items = [(to_image(r["image"]), prompts.OCR_PAGE[0]) for _, r in sub.iterrows()]
        preds = generate(model, processor, items, args.max_new_tokens, args.bs)
        by = defaultdict(list)
        rows = []
        for (_, r), p in zip(sub.iterrows(), preds):
            c = cer(p, r["text"], args.ocr_chars)
            by[r["language"]].append(c)
            rows.append({"id": r["id"], "language": r["language"], "cer": c, "pred": p[:400], "ref": r["text"][:400]})
        res["tasks"]["ocr"] = {
            "cer_mean": sum(sum(v) for v in by.values()) / sum(len(v) for v in by.values()),
            "cer_by_language": {k: sum(v) / len(v) for k, v in sorted(by.items())},
            "n": len(rows), "rows": rows,
        }
        print(f"[eval] OCR CER mean = {res['tasks']['ocr']['cer_mean']:.4f}", flush=True)

    if "mmt" in args.tasks:
        import sacrebleu
        df = pd.read_parquet(next((bench / "mmt").glob("*.parquet")))
        items, refs, langs = [], [], []
        for i, (_, r) in enumerate(df.iterrows()):
            lang = MMT_LANGS[i % len(MMT_LANGS)]
            items.append((to_image(r["image"]),
                          f"Translate the following English text into {lang}. Reply with the translation only.\n\n{r['English']}"))
            refs.append(r[lang])
            langs.append(lang)
        print(f"[eval] mmt: {len(items)} items", flush=True)
        preds = generate(model, processor, items, 256, args.bs)
        by = defaultdict(lambda: ([], []))
        for p, ref, lang in zip(preds, refs, langs):
            by[lang][0].append(p.strip())
            by[lang][1].append(ref)
        score = {k: sacrebleu.corpus_chrf(v[0], [v[1]]).score for k, v in sorted(by.items())}
        allp, allr = [p.strip() for p in preds], list(refs)
        res["tasks"]["mmt"] = {"chrf_mean": sacrebleu.corpus_chrf(allp, [allr]).score, "chrf_by_language": score,
                               "n": len(items)}
        print(f"[eval] MMT chrF = {res['tasks']['mmt']['chrf_mean']:.2f}", flush=True)

    Path(args.out).write_text(json.dumps(res, ensure_ascii=False, indent=1))
    print(f"[eval] wrote {args.out}  ({time.time() - t0:.0f}s)", flush=True)


if __name__ == "__main__":
    main()