Instructions to use srivassid/drishti with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use srivassid/drishti with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("google/gemma-4-12B-it") model = PeftModel.from_pretrained(base_model, "srivassid/drishti") - Notebooks
- Google Colab
- Kaggle
Download code/eval_bench.py from srivassid/drishti: direct link, hf CLI and curl.
- Browser
- Download file 7.4 kB
-
https://huggingface.co/srivassid/drishti/resolve/main/code/eval_bench.py
- Command line
-
hf download hf://srivassid/drishti/code/eval_bench.py
-
curl -L -o eval_bench.py https://huggingface.co/srivassid/drishti/resolve/main/code/eval_bench.py
7.4 kB
| """Evaluate base vs fine-tuned Gemma on IndicVisionBench (a benchmark that is NOT in the training mix). | |
| CUDA_VISIBLE_DEVICES=1 python -m vlmin.eval_bench --bench /mnt/hdd/vlmin/bench/IndicVisionBench \ | |
| --out base.json # base model | |
| CUDA_VISIBLE_DEVICES=2 python -m vlmin.eval_bench ... --adapter runs/<run>/final --out tuned.json | |
| Tasks (objective metrics only, no LLM judge): | |
| ocr page transcription, scored by CER on the first --ocr-chars characters (prediction and reference are | |
| both cut to that length, so a token limit doesn't penalise long pages). Lower is better. | |
| mmt translate the English caption of an image into one Indic language, scored by chrF. Higher is better. | |
| Prompts reuse the wording seen in training (prompts.OCR_PAGE[0]) so the comparison is not about prompt format. | |
| """ | |
| import argparse | |
| import io | |
| import json | |
| import os | |
| import random | |
| import time | |
| from collections import defaultdict | |
| from pathlib import Path | |
| os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") | |
| os.environ.setdefault("CUDA_DEVICE_ORDER", "PCI_BUS_ID") # same GPU numbering as nvidia-smi and train.py | |
| os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") | |
| import pandas as pd # noqa: E402 | |
| import torch # noqa: E402 | |
| from PIL import Image # noqa: E402 | |
| from transformers import AutoModelForImageTextToText, AutoProcessor, BitsAndBytesConfig # noqa: E402 | |
| from . import prompts # noqa: E402 | |
| MMT_LANGS = ["Hindi", "Bengali", "Gujarati", "Kannada", "Malayalam", "Marathi", "Odia", "Punjabi", "Tamil", "Telugu"] | |
| def load_model(model_id, adapter): | |
| bnb = BitsAndBytesConfig( | |
| load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, | |
| bnb_4bit_compute_dtype=torch.bfloat16, | |
| llm_int8_skip_modules=["vision_tower", "multi_modal_projector", "embed_vision", "embed_audio", | |
| "vision_embedder", "lm_head"], | |
| ) | |
| model = AutoModelForImageTextToText.from_pretrained( | |
| model_id, quantization_config=bnb, dtype=torch.bfloat16, attn_implementation="sdpa", device_map={"": 0}) | |
| if adapter: | |
| from peft import PeftModel | |
| model = PeftModel.from_pretrained(model, adapter) | |
| model.eval() | |
| return model | |
| def to_image(cell, max_side=1344): | |
| im = Image.open(io.BytesIO(cell["bytes"])).convert("RGB") | |
| im.thumbnail((max_side, max_side), Image.LANCZOS) | |
| return im | |
| def generate(model, processor, items, max_new_tokens, bs): | |
| """items: list of (PIL image, prompt text). Greedy, left-padded batches. Returns list of strings.""" | |
| processor.tokenizer.padding_side = "left" | |
| outs = [] | |
| for i in range(0, len(items), bs): | |
| chunk = items[i:i + bs] | |
| msgs = [[{"role": "user", "content": [{"type": "image", "image": im}, {"type": "text", "text": p}]}] | |
| for im, p in chunk] | |
| enc = processor.apply_chat_template(msgs, tokenize=True, return_dict=True, return_tensors="pt", | |
| padding=True, add_generation_prompt=True) | |
| enc = {k: v.to("cuda") for k, v in enc.items()} | |
| enc["pixel_values"] = enc["pixel_values"].to(torch.bfloat16) | |
| n_in = enc["input_ids"].shape[1] | |
| gen = model.generate(**enc, max_new_tokens=max_new_tokens, do_sample=False) | |
| outs += processor.batch_decode(gen[:, n_in:], skip_special_tokens=True) | |
| print(f" generated {min(i + bs, len(items))}/{len(items)}", flush=True) | |
| return outs | |
| def cer(pred, ref, n): | |
| from rapidfuzz.distance import Levenshtein | |
| ref, pred = ref[:n], pred[:n] | |
| return Levenshtein.distance(pred, ref) / max(1, len(ref)) | |
| def main(): | |
| ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| ap.add_argument("--bench", required=True) | |
| ap.add_argument("--model", default="/run/media/sid/disk/VLM-IN/models/gemma-4-12B-it") | |
| ap.add_argument("--adapter") | |
| ap.add_argument("--out", required=True) | |
| ap.add_argument("--tasks", default="ocr,mmt") | |
| ap.add_argument("--ocr-per-lang", type=int, default=20) | |
| ap.add_argument("--ocr-chars", type=int, default=600) | |
| ap.add_argument("--max-new-tokens", type=int, default=768) | |
| ap.add_argument("--bs", type=int, default=4) | |
| ap.add_argument("--seed", type=int, default=0) | |
| args = ap.parse_args() | |
| bench = Path(args.bench) | |
| t0 = time.time() | |
| processor = AutoProcessor.from_pretrained(args.model) | |
| model = load_model(args.model, args.adapter) | |
| print(f"[eval] model loaded in {time.time() - t0:.0f}s adapter={args.adapter}", flush=True) | |
| res = {"adapter": args.adapter, "tasks": {}} | |
| if "ocr" in args.tasks: | |
| df = pd.concat([pd.read_parquet(p) for p in sorted((bench / "ocr").glob("*.parquet"))], ignore_index=True) | |
| rng = random.Random(args.seed) | |
| pick = [] | |
| for lang, g in df.groupby("language"): | |
| idx = list(g.index) | |
| rng.shuffle(idx) | |
| pick += idx[:args.ocr_per_lang] | |
| sub = df.loc[sorted(pick)] | |
| print(f"[eval] ocr: {len(sub)} pages, languages={sorted(sub['language'].unique())}", flush=True) | |
| items = [(to_image(r["image"]), prompts.OCR_PAGE[0]) for _, r in sub.iterrows()] | |
| preds = generate(model, processor, items, args.max_new_tokens, args.bs) | |
| by = defaultdict(list) | |
| rows = [] | |
| for (_, r), p in zip(sub.iterrows(), preds): | |
| c = cer(p, r["text"], args.ocr_chars) | |
| by[r["language"]].append(c) | |
| rows.append({"id": r["id"], "language": r["language"], "cer": c, "pred": p[:400], "ref": r["text"][:400]}) | |
| res["tasks"]["ocr"] = { | |
| "cer_mean": sum(sum(v) for v in by.values()) / sum(len(v) for v in by.values()), | |
| "cer_by_language": {k: sum(v) / len(v) for k, v in sorted(by.items())}, | |
| "n": len(rows), "rows": rows, | |
| } | |
| print(f"[eval] OCR CER mean = {res['tasks']['ocr']['cer_mean']:.4f}", flush=True) | |
| if "mmt" in args.tasks: | |
| import sacrebleu | |
| df = pd.read_parquet(next((bench / "mmt").glob("*.parquet"))) | |
| items, refs, langs = [], [], [] | |
| for i, (_, r) in enumerate(df.iterrows()): | |
| lang = MMT_LANGS[i % len(MMT_LANGS)] | |
| items.append((to_image(r["image"]), | |
| f"Translate the following English text into {lang}. Reply with the translation only.\n\n{r['English']}")) | |
| refs.append(r[lang]) | |
| langs.append(lang) | |
| print(f"[eval] mmt: {len(items)} items", flush=True) | |
| preds = generate(model, processor, items, 256, args.bs) | |
| by = defaultdict(lambda: ([], [])) | |
| for p, ref, lang in zip(preds, refs, langs): | |
| by[lang][0].append(p.strip()) | |
| by[lang][1].append(ref) | |
| score = {k: sacrebleu.corpus_chrf(v[0], [v[1]]).score for k, v in sorted(by.items())} | |
| allp, allr = [p.strip() for p in preds], list(refs) | |
| res["tasks"]["mmt"] = {"chrf_mean": sacrebleu.corpus_chrf(allp, [allr]).score, "chrf_by_language": score, | |
| "n": len(items)} | |
| print(f"[eval] MMT chrF = {res['tasks']['mmt']['chrf_mean']:.2f}", flush=True) | |
| Path(args.out).write_text(json.dumps(res, ensure_ascii=False, indent=1)) | |
| print(f"[eval] wrote {args.out} ({time.time() - t0:.0f}s)", flush=True) | |
| if __name__ == "__main__": | |
| main() | |