Spaces:
Running on Zero
Running on Zero
Download measure_reference.py from Gabriel/diffu-htr-speed: direct link, hf CLI and curl.
- Browser
- Download file 6.96 kB
-
https://huggingface.co/spaces/Gabriel/diffu-htr-speed/resolve/main/measure_reference.py
- Command line
-
hf download hf://spaces/Gabriel/diffu-htr-speed/measure_reference.py
-
curl -L -o measure_reference.py https://huggingface.co/spaces/Gabriel/diffu-htr-speed/resolve/main/measure_reference.py
6.96 kB
| """The Space's reference timings: each example region read on this machine by the Swedish Lion TrOCR (transformers, | |
| its lines one after the other and as one batch, as the app runs it) and by the reader in vLLM, alone and with its | |
| DSpark draft, one request at a time. The app shows them beside its live race, which runs in PyTorch because ZeroGPU | |
| cannot host vLLM. | |
| Serve the reader twice first, as the draft's card says (prefix caching off), the second server once the first | |
| answers, on a GPU nothing else is using: | |
| vllm serve READER --served-model-name htr ... --port 8057 | |
| vllm serve READER --served-model-name htr ... --port 8056 --speculative-config '{"method": "dspark", ...}' | |
| python measure_reference.py --examples examples --gpu "RTX PRO 6000 Blackwell" --out reference.json | |
| Each reading is timed from the encoded images to the reading: the median of RUNS runs after one that is not timed. | |
| vLLM keeps the processed form of an image it has seen, so every request sends the region with one pixel, drawn at | |
| random, changed by one level: the region is new to the server each time, as a region is in production. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import base64 | |
| import io | |
| import json | |
| import secrets | |
| import statistics | |
| import time | |
| from collections.abc import Callable | |
| from datetime import date | |
| from pathlib import Path | |
| from typing import Any | |
| import httpx | |
| import torch | |
| from PIL import Image | |
| from transformers import TrOCRProcessor, VisionEncoderDecoderModel | |
| from reference import Reference, RegionTimes | |
| SAMPLING = {"temperature": 0.01, "top_p": 0.001, "top_k": 1, "repetition_penalty": 1.0} # OpenBMB's, near greedy | |
| IMAGES = {"downsample_mode": "4x", "max_slice_nums": 9} | |
| def median_seconds(read: Callable[[int], None], runs: int) -> float: | |
| """The median time of `runs` calls of `read`, after one that is not timed; each call gets its own number.""" | |
| read(0) | |
| times = [] | |
| for run in range(1, runs + 1): | |
| started = time.perf_counter() | |
| read(run) | |
| times.append(time.perf_counter() - started) | |
| return round(statistics.median(times), 3) | |
| def requests_for(region: Path, count: int) -> list[dict[str, Any]]: | |
| """`count` requests for one region, each with one pixel, drawn at random, changed by one level.""" | |
| image = Image.open(region).convert("RGB") | |
| pixels = image.tobytes() | |
| bodies = [] | |
| for _ in range(count): | |
| changed = bytearray(pixels) | |
| changed[3 * secrets.randbelow(image.width * image.height)] ^= 1 # a pixel's red | |
| buffer = io.BytesIO() | |
| Image.frombytes("RGB", image.size, bytes(changed)).save(buffer, format="PNG") | |
| url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() | |
| content = [{"type": "image_url", "image_url": {"url": url}}, {"type": "text", "text": "Text Recognition:"}] | |
| bodies.append( | |
| { | |
| "model": "htr", | |
| **SAMPLING, | |
| "max_tokens": 4096, | |
| "chat_template_kwargs": {"enable_thinking": False}, | |
| "mm_processor_kwargs": IMAGES, | |
| "messages": [{"role": "user", "content": content}], | |
| } | |
| ) | |
| return bodies | |
| class Vllm: | |
| """One vLLM server, asked for one region at a time; it keeps what the region's requests answered.""" | |
| def __init__(self, endpoint: str, client: httpx.Client) -> None: | |
| self.url = f"{endpoint}/v1/chat/completions" | |
| self.client = client | |
| self.texts: set[str] = set() | |
| self.tokens = 0 | |
| def read(self, body: dict[str, Any]) -> None: | |
| reply = self.client.post(self.url, json=body) | |
| reply.raise_for_status() | |
| answer = reply.json() | |
| self.texts.add(answer["choices"][0]["message"]["content"]) | |
| self.tokens = answer["usage"]["completion_tokens"] | |
| def main() -> None: | |
| parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) | |
| parser.add_argument("--examples", type=Path, required=True, help="the folder build_examples.py wrote") | |
| parser.add_argument("--trocr", default="Gabriel/swedish-lion-trocr-large-handwritten-char") | |
| parser.add_argument("--alone", default="http://localhost:8057", help="vLLM serving the reader alone") | |
| parser.add_argument("--draft", default="http://localhost:8056", help="vLLM serving the reader with its draft") | |
| parser.add_argument("--gpu", required=True, help="the GPU's name, as the app will show it") | |
| parser.add_argument("--runs", type=int, default=5) | |
| parser.add_argument("--out", type=Path, required=True) | |
| args = parser.parse_args() | |
| processor = TrOCRProcessor.from_pretrained(args.trocr) | |
| trocr = VisionEncoderDecoderModel.from_pretrained(args.trocr).to("cuda").eval() | |
| def read_lines(files: list[bytes], batch: int) -> None: | |
| lines = [Image.open(io.BytesIO(data)).convert("RGB") for data in files] | |
| for first in range(0, len(lines), batch): | |
| pixels = processor(images=lines[first : first + batch], return_tensors="pt").pixel_values.to("cuda") | |
| with torch.inference_mode(): | |
| trocr.generate( | |
| pixels, | |
| decoder_start_token_id=trocr.config.decoder_start_token_id, | |
| interpolate_pos_encoding=True, | |
| num_beams=1, | |
| max_new_tokens=128, | |
| ) | |
| torch.cuda.synchronize() | |
| regions = {} | |
| with httpx.Client(timeout=600) as client: | |
| for example in json.loads((args.examples / "examples.json").read_text(encoding="utf-8")): | |
| folder = args.examples / example["id"] | |
| files = [(folder / f"line_{i:02d}.jpg").read_bytes() for i in range(len(example["lines"]))] | |
| bodies = requests_for(folder / "region.png", args.runs + 1) | |
| alone, draft = Vllm(args.alone, client), Vllm(args.draft, client) | |
| times = RegionTimes( | |
| lines=len(files), | |
| trocr_single=median_seconds(lambda _, files=files: read_lines(files, 1), args.runs), | |
| trocr_batch=median_seconds(lambda _, files=files: read_lines(files, len(files)), args.runs), | |
| vllm=median_seconds(lambda run, alone=alone, bodies=bodies: alone.read(bodies[run]), args.runs), | |
| vllm_dspark=median_seconds(lambda run, draft=draft, bodies=bodies: draft.read(bodies[run]), args.runs), | |
| tokens=alone.tokens, | |
| ) | |
| regions[example["id"]] = times | |
| same = "the same text" if alone.texts == draft.texts and len(alone.texts) == 1 else "texts differ" | |
| print(example["id"], times.model_dump(), f"{len(alone.texts)} and {len(draft.texts)} readings, {same}") | |
| reference = Reference(gpu=args.gpu, measured=date.today().isoformat(), runs=args.runs, regions=regions) | |
| args.out.write_text(reference.model_dump_json(indent=1) + "\n", encoding="utf-8") | |
| if __name__ == "__main__": | |
| main() | |