Spaces:
Running on Zero
Running on Zero
File size: 4,233 Bytes
8583929 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | """Check dspark_hf on a GPU: the reader alone and with its draft read the same lines and regions, compared token for
token, and timed; the first lines are also compared with transformers' own `generate`.
python check_decode.py READER DRAFT LINES.jsonl REGIONS.jsonl [--lines 20] [--regions 12] [--cache 4096]
READER and DRAFT are Hub ids or folders; the sets are `htr-eval`'s lines files (id, image, text).
"""
from __future__ import annotations
import argparse
import json
import statistics
import time
from pathlib import Path
from typing import Any
import torch
from huggingface_hub import snapshot_download
from PIL import Image
from transformers import AutoModelForImageTextToText, AutoProcessor
import dspark_hf as D
IMAGES = {"downsample_mode": "4x", "max_slice_nums": 9}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("reader")
parser.add_argument("draft")
parser.add_argument("lines", type=Path)
parser.add_argument("regions", type=Path)
parser.add_argument("--lines", dest="n_lines", type=int, default=20)
parser.add_argument("--regions", dest="n_regions", type=int, default=12)
parser.add_argument("--cache", type=int, default=4096, help="the reader's cache length (prompt and reading)")
args = parser.parse_args()
processor = AutoProcessor.from_pretrained(args.reader)
model = AutoModelForImageTextToText.from_pretrained(args.reader, dtype=torch.bfloat16).to("cuda").eval()
folder = args.draft if Path(args.draft).is_dir() else snapshot_download(args.draft)
draft = D.DSparkDraft.from_folder(folder, model.get_input_embeddings()).to("cuda", torch.bfloat16).eval()
eos_ids = model.generation_config.eos_token_id
eos = set(eos_ids) if isinstance(eos_ids, list) else {int(eos_ids)}
def inputs(row: dict[str, Any]) -> dict[str, Any]:
content = [{"type": "image", "image": Image.open(row["image"]).convert("RGB")}]
content.append({"type": "text", "text": "Text Recognition:"})
batch = processor.apply_chat_template(
[{"role": "user", "content": content}],
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
enable_thinking=False,
processor_kwargs=IMAGES,
)
return dict(batch.to("cuda")) | {"downsample_mode": IMAGES["downsample_mode"]}
def rows(path: Path, n: int) -> list[dict[str, Any]]:
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()[:n]]
lines, regions = rows(args.lines, args.n_lines), rows(args.regions, args.n_regions)
started = time.perf_counter()
engine = D.FastDSpark(model, draft, args.cache)
engine.warm(inputs(regions[0]), eos)
print(f"graphs captured in {time.perf_counter() - started:.1f} s")
with torch.inference_mode():
same = 0
for row in lines[:5]:
x = inputs(row)
reference = model.generate(**x, max_new_tokens=512, do_sample=False)[0, x["input_ids"].shape[1] :].tolist()
same += D.greedy(engine.reader, x, 512, eos).tokens == reference
print(f"the reader alone on graphs against transformers' generate: {same} of {min(5, len(lines))} identical")
for name, sample, max_new in (("lines", lines, 512), ("regions", regions, 4096)):
alone, drafted = [], []
for row in sample:
x = inputs(row)
alone.append(D.greedy(engine.reader, x, max_new, eos))
drafted.append(engine.decode(x, max_new, eos))
a, d = sum(r.seconds for r in alone), sum(r.seconds for r in drafted)
identical = sum(x.tokens == y.tokens for x, y in zip(alone, drafted, strict=True))
acceptance = statistics.fmean(r.acceptance_length for r in drafted if r.cycles)
tokens = sum(len(r.tokens) for r in alone)
print(
f"{name}: {len(sample)} read, {tokens} tokens; reader alone {a:.2f} s, with the draft {d:.2f} s "
f"({a / d:.2f}x); identical {identical} of {len(sample)}; acceptance length {acceptance:.2f}"
)
if __name__ == "__main__":
main()
|