"""Evaluate base vs fine-tuned Gemma on IndicVisionBench (a benchmark that is NOT in the training mix). CUDA_VISIBLE_DEVICES=1 python -m vlmin.eval_bench --bench /mnt/hdd/vlmin/bench/IndicVisionBench \ --out base.json # base model CUDA_VISIBLE_DEVICES=2 python -m vlmin.eval_bench ... --adapter runs//final --out tuned.json Tasks (objective metrics only, no LLM judge): ocr page transcription, scored by CER on the first --ocr-chars characters (prediction and reference are both cut to that length, so a token limit doesn't penalise long pages). Lower is better. mmt translate the English caption of an image into one Indic language, scored by chrF. Higher is better. Prompts reuse the wording seen in training (prompts.OCR_PAGE[0]) so the comparison is not about prompt format. """ import argparse import io import json import os import random import time from collections import defaultdict from pathlib import Path os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID") # same GPU numbering as nvidia-smi and train.py os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import pandas as pd # noqa: E402 import torch # noqa: E402 from PIL import Image # noqa: E402 from transformers import AutoModelForImageTextToText, AutoProcessor, BitsAndBytesConfig # noqa: E402 from . import prompts # noqa: E402 MMT_LANGS = ["Hindi", "Bengali", "Gujarati", "Kannada", "Malayalam", "Marathi", "Odia", "Punjabi", "Tamil", "Telugu"] def load_model(model_id, adapter): bnb = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.bfloat16, llm_int8_skip_modules=["vision_tower", "multi_modal_projector", "embed_vision", "embed_audio", "vision_embedder", "lm_head"], ) model = AutoModelForImageTextToText.from_pretrained( model_id, quantization_config=bnb, dtype=torch.bfloat16, attn_implementation="sdpa", device_map={"": 0}) if adapter: from peft import PeftModel model = PeftModel.from_pretrained(model, adapter) model.eval() return model def to_image(cell, max_side=1344): im = Image.open(io.BytesIO(cell["bytes"])).convert("RGB") im.thumbnail((max_side, max_side), Image.LANCZOS) return im @torch.no_grad() def generate(model, processor, items, max_new_tokens, bs): """items: list of (PIL image, prompt text). Greedy, left-padded batches. Returns list of strings.""" processor.tokenizer.padding_side = "left" outs = [] for i in range(0, len(items), bs): chunk = items[i:i + bs] msgs = [[{"role": "user", "content": [{"type": "image", "image": im}, {"type": "text", "text": p}]}] for im, p in chunk] enc = processor.apply_chat_template(msgs, tokenize=True, return_dict=True, return_tensors="pt", padding=True, add_generation_prompt=True) enc = {k: v.to("cuda") for k, v in enc.items()} enc["pixel_values"] = enc["pixel_values"].to(torch.bfloat16) n_in = enc["input_ids"].shape[1] gen = model.generate(**enc, max_new_tokens=max_new_tokens, do_sample=False) outs += processor.batch_decode(gen[:, n_in:], skip_special_tokens=True) print(f" generated {min(i + bs, len(items))}/{len(items)}", flush=True) return outs def cer(pred, ref, n): from rapidfuzz.distance import Levenshtein ref, pred = ref[:n], pred[:n] return Levenshtein.distance(pred, ref) / max(1, len(ref)) def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--bench", required=True) ap.add_argument("--model", default="/run/media/sid/disk/VLM-IN/models/gemma-4-12B-it") ap.add_argument("--adapter") ap.add_argument("--out", required=True) ap.add_argument("--tasks", default="ocr,mmt") ap.add_argument("--ocr-per-lang", type=int, default=20) ap.add_argument("--ocr-chars", type=int, default=600) ap.add_argument("--max-new-tokens", type=int, default=768) ap.add_argument("--bs", type=int, default=4) ap.add_argument("--seed", type=int, default=0) args = ap.parse_args() bench = Path(args.bench) t0 = time.time() processor = AutoProcessor.from_pretrained(args.model) model = load_model(args.model, args.adapter) print(f"[eval] model loaded in {time.time() - t0:.0f}s adapter={args.adapter}", flush=True) res = {"adapter": args.adapter, "tasks": {}} if "ocr" in args.tasks: df = pd.concat([pd.read_parquet(p) for p in sorted((bench / "ocr").glob("*.parquet"))], ignore_index=True) rng = random.Random(args.seed) pick = [] for lang, g in df.groupby("language"): idx = list(g.index) rng.shuffle(idx) pick += idx[:args.ocr_per_lang] sub = df.loc[sorted(pick)] print(f"[eval] ocr: {len(sub)} pages, languages={sorted(sub['language'].unique())}", flush=True) items = [(to_image(r["image"]), prompts.OCR_PAGE[0]) for _, r in sub.iterrows()] preds = generate(model, processor, items, args.max_new_tokens, args.bs) by = defaultdict(list) rows = [] for (_, r), p in zip(sub.iterrows(), preds): c = cer(p, r["text"], args.ocr_chars) by[r["language"]].append(c) rows.append({"id": r["id"], "language": r["language"], "cer": c, "pred": p[:400], "ref": r["text"][:400]}) res["tasks"]["ocr"] = { "cer_mean": sum(sum(v) for v in by.values()) / sum(len(v) for v in by.values()), "cer_by_language": {k: sum(v) / len(v) for k, v in sorted(by.items())}, "n": len(rows), "rows": rows, } print(f"[eval] OCR CER mean = {res['tasks']['ocr']['cer_mean']:.4f}", flush=True) if "mmt" in args.tasks: import sacrebleu df = pd.read_parquet(next((bench / "mmt").glob("*.parquet"))) items, refs, langs = [], [], [] for i, (_, r) in enumerate(df.iterrows()): lang = MMT_LANGS[i % len(MMT_LANGS)] items.append((to_image(r["image"]), f"Translate the following English text into {lang}. Reply with the translation only.\n\n{r['English']}")) refs.append(r[lang]) langs.append(lang) print(f"[eval] mmt: {len(items)} items", flush=True) preds = generate(model, processor, items, 256, args.bs) by = defaultdict(lambda: ([], [])) for p, ref, lang in zip(preds, refs, langs): by[lang][0].append(p.strip()) by[lang][1].append(ref) score = {k: sacrebleu.corpus_chrf(v[0], [v[1]]).score for k, v in sorted(by.items())} allp, allr = [p.strip() for p in preds], list(refs) res["tasks"]["mmt"] = {"chrf_mean": sacrebleu.corpus_chrf(allp, [allr]).score, "chrf_by_language": score, "n": len(items)} print(f"[eval] MMT chrF = {res['tasks']['mmt']['chrf_mean']:.2f}", flush=True) Path(args.out).write_text(json.dumps(res, ensure_ascii=False, indent=1)) print(f"[eval] wrote {args.out} ({time.time() - t0:.0f}s)", flush=True) if __name__ == "__main__": main()