diffu-htr-speed / measure_reference.py
Gabriel's picture
One stream each: TrOCR reads its lines one by one (its one-batch time beside it), the replay starts all three together, our server's vLLM timings under the race, one region box, Jev's rows folded with marked errors
b0dacaa verified
Raw History Blame Contribute Delete
6.96 kB
"""The Space's reference timings: each example region read on this machine by the Swedish Lion TrOCR (transformers,
its lines one after the other and as one batch, as the app runs it) and by the reader in vLLM, alone and with its
DSpark draft, one request at a time. The app shows them beside its live race, which runs in PyTorch because ZeroGPU
cannot host vLLM.
Serve the reader twice first, as the draft's card says (prefix caching off), the second server once the first
answers, on a GPU nothing else is using:
vllm serve READER --served-model-name htr ... --port 8057
vllm serve READER --served-model-name htr ... --port 8056 --speculative-config '{"method": "dspark", ...}'
python measure_reference.py --examples examples --gpu "RTX PRO 6000 Blackwell" --out reference.json
Each reading is timed from the encoded images to the reading: the median of RUNS runs after one that is not timed.
vLLM keeps the processed form of an image it has seen, so every request sends the region with one pixel, drawn at
random, changed by one level: the region is new to the server each time, as a region is in production.
"""
from __future__ import annotations
import argparse
import base64
import io
import json
import secrets
import statistics
import time
from collections.abc import Callable
from datetime import date
from pathlib import Path
from typing import Any
import httpx
import torch
from PIL import Image
from transformers import TrOCRProcessor, VisionEncoderDecoderModel
from reference import Reference, RegionTimes
SAMPLING = {"temperature": 0.01, "top_p": 0.001, "top_k": 1, "repetition_penalty": 1.0} # OpenBMB's, near greedy
IMAGES = {"downsample_mode": "4x", "max_slice_nums": 9}
def median_seconds(read: Callable[[int], None], runs: int) -> float:
"""The median time of `runs` calls of `read`, after one that is not timed; each call gets its own number."""
read(0)
times = []
for run in range(1, runs + 1):
started = time.perf_counter()
read(run)
times.append(time.perf_counter() - started)
return round(statistics.median(times), 3)
def requests_for(region: Path, count: int) -> list[dict[str, Any]]:
"""`count` requests for one region, each with one pixel, drawn at random, changed by one level."""
image = Image.open(region).convert("RGB")
pixels = image.tobytes()
bodies = []
for _ in range(count):
changed = bytearray(pixels)
changed[3 * secrets.randbelow(image.width * image.height)] ^= 1 # a pixel's red
buffer = io.BytesIO()
Image.frombytes("RGB", image.size, bytes(changed)).save(buffer, format="PNG")
url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
content = [{"type": "image_url", "image_url": {"url": url}}, {"type": "text", "text": "Text Recognition:"}]
bodies.append(
{
"model": "htr",
**SAMPLING,
"max_tokens": 4096,
"chat_template_kwargs": {"enable_thinking": False},
"mm_processor_kwargs": IMAGES,
"messages": [{"role": "user", "content": content}],
}
)
return bodies
class Vllm:
"""One vLLM server, asked for one region at a time; it keeps what the region's requests answered."""
def __init__(self, endpoint: str, client: httpx.Client) -> None:
self.url = f"{endpoint}/v1/chat/completions"
self.client = client
self.texts: set[str] = set()
self.tokens = 0
def read(self, body: dict[str, Any]) -> None:
reply = self.client.post(self.url, json=body)
reply.raise_for_status()
answer = reply.json()
self.texts.add(answer["choices"][0]["message"]["content"])
self.tokens = answer["usage"]["completion_tokens"]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--examples", type=Path, required=True, help="the folder build_examples.py wrote")
parser.add_argument("--trocr", default="Gabriel/swedish-lion-trocr-large-handwritten-char")
parser.add_argument("--alone", default="http://localhost:8057", help="vLLM serving the reader alone")
parser.add_argument("--draft", default="http://localhost:8056", help="vLLM serving the reader with its draft")
parser.add_argument("--gpu", required=True, help="the GPU's name, as the app will show it")
parser.add_argument("--runs", type=int, default=5)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
processor = TrOCRProcessor.from_pretrained(args.trocr)
trocr = VisionEncoderDecoderModel.from_pretrained(args.trocr).to("cuda").eval()
def read_lines(files: list[bytes], batch: int) -> None:
lines = [Image.open(io.BytesIO(data)).convert("RGB") for data in files]
for first in range(0, len(lines), batch):
pixels = processor(images=lines[first : first + batch], return_tensors="pt").pixel_values.to("cuda")
with torch.inference_mode():
trocr.generate(
pixels,
decoder_start_token_id=trocr.config.decoder_start_token_id,
interpolate_pos_encoding=True,
num_beams=1,
max_new_tokens=128,
)
torch.cuda.synchronize()
regions = {}
with httpx.Client(timeout=600) as client:
for example in json.loads((args.examples / "examples.json").read_text(encoding="utf-8")):
folder = args.examples / example["id"]
files = [(folder / f"line_{i:02d}.jpg").read_bytes() for i in range(len(example["lines"]))]
bodies = requests_for(folder / "region.png", args.runs + 1)
alone, draft = Vllm(args.alone, client), Vllm(args.draft, client)
times = RegionTimes(
lines=len(files),
trocr_single=median_seconds(lambda _, files=files: read_lines(files, 1), args.runs),
trocr_batch=median_seconds(lambda _, files=files: read_lines(files, len(files)), args.runs),
vllm=median_seconds(lambda run, alone=alone, bodies=bodies: alone.read(bodies[run]), args.runs),
vllm_dspark=median_seconds(lambda run, draft=draft, bodies=bodies: draft.read(bodies[run]), args.runs),
tokens=alone.tokens,
)
regions[example["id"]] = times
same = "the same text" if alone.texts == draft.texts and len(alone.texts) == 1 else "texts differ"
print(example["id"], times.model_dump(), f"{len(alone.texts)} and {len(draft.texts)} readings, {same}")
reference = Reference(gpu=args.gpu, measured=date.today().isoformat(), runs=args.runs, regions=regions)
args.out.write_text(reference.model_dump_json(indent=1) + "\n", encoding="utf-8")
if __name__ == "__main__":
main()