Spaces:
Running on Zero
Running on Zero
File size: 6,962 Bytes
b0dacaa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | """The Space's reference timings: each example region read on this machine by the Swedish Lion TrOCR (transformers,
its lines one after the other and as one batch, as the app runs it) and by the reader in vLLM, alone and with its
DSpark draft, one request at a time. The app shows them beside its live race, which runs in PyTorch because ZeroGPU
cannot host vLLM.
Serve the reader twice first, as the draft's card says (prefix caching off), the second server once the first
answers, on a GPU nothing else is using:
vllm serve READER --served-model-name htr ... --port 8057
vllm serve READER --served-model-name htr ... --port 8056 --speculative-config '{"method": "dspark", ...}'
python measure_reference.py --examples examples --gpu "RTX PRO 6000 Blackwell" --out reference.json
Each reading is timed from the encoded images to the reading: the median of RUNS runs after one that is not timed.
vLLM keeps the processed form of an image it has seen, so every request sends the region with one pixel, drawn at
random, changed by one level: the region is new to the server each time, as a region is in production.
"""
from __future__ import annotations
import argparse
import base64
import io
import json
import secrets
import statistics
import time
from collections.abc import Callable
from datetime import date
from pathlib import Path
from typing import Any
import httpx
import torch
from PIL import Image
from transformers import TrOCRProcessor, VisionEncoderDecoderModel
from reference import Reference, RegionTimes
SAMPLING = {"temperature": 0.01, "top_p": 0.001, "top_k": 1, "repetition_penalty": 1.0} # OpenBMB's, near greedy
IMAGES = {"downsample_mode": "4x", "max_slice_nums": 9}
def median_seconds(read: Callable[[int], None], runs: int) -> float:
"""The median time of `runs` calls of `read`, after one that is not timed; each call gets its own number."""
read(0)
times = []
for run in range(1, runs + 1):
started = time.perf_counter()
read(run)
times.append(time.perf_counter() - started)
return round(statistics.median(times), 3)
def requests_for(region: Path, count: int) -> list[dict[str, Any]]:
"""`count` requests for one region, each with one pixel, drawn at random, changed by one level."""
image = Image.open(region).convert("RGB")
pixels = image.tobytes()
bodies = []
for _ in range(count):
changed = bytearray(pixels)
changed[3 * secrets.randbelow(image.width * image.height)] ^= 1 # a pixel's red
buffer = io.BytesIO()
Image.frombytes("RGB", image.size, bytes(changed)).save(buffer, format="PNG")
url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
content = [{"type": "image_url", "image_url": {"url": url}}, {"type": "text", "text": "Text Recognition:"}]
bodies.append(
{
"model": "htr",
**SAMPLING,
"max_tokens": 4096,
"chat_template_kwargs": {"enable_thinking": False},
"mm_processor_kwargs": IMAGES,
"messages": [{"role": "user", "content": content}],
}
)
return bodies
class Vllm:
"""One vLLM server, asked for one region at a time; it keeps what the region's requests answered."""
def __init__(self, endpoint: str, client: httpx.Client) -> None:
self.url = f"{endpoint}/v1/chat/completions"
self.client = client
self.texts: set[str] = set()
self.tokens = 0
def read(self, body: dict[str, Any]) -> None:
reply = self.client.post(self.url, json=body)
reply.raise_for_status()
answer = reply.json()
self.texts.add(answer["choices"][0]["message"]["content"])
self.tokens = answer["usage"]["completion_tokens"]
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--examples", type=Path, required=True, help="the folder build_examples.py wrote")
parser.add_argument("--trocr", default="Gabriel/swedish-lion-trocr-large-handwritten-char")
parser.add_argument("--alone", default="http://localhost:8057", help="vLLM serving the reader alone")
parser.add_argument("--draft", default="http://localhost:8056", help="vLLM serving the reader with its draft")
parser.add_argument("--gpu", required=True, help="the GPU's name, as the app will show it")
parser.add_argument("--runs", type=int, default=5)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
processor = TrOCRProcessor.from_pretrained(args.trocr)
trocr = VisionEncoderDecoderModel.from_pretrained(args.trocr).to("cuda").eval()
def read_lines(files: list[bytes], batch: int) -> None:
lines = [Image.open(io.BytesIO(data)).convert("RGB") for data in files]
for first in range(0, len(lines), batch):
pixels = processor(images=lines[first : first + batch], return_tensors="pt").pixel_values.to("cuda")
with torch.inference_mode():
trocr.generate(
pixels,
decoder_start_token_id=trocr.config.decoder_start_token_id,
interpolate_pos_encoding=True,
num_beams=1,
max_new_tokens=128,
)
torch.cuda.synchronize()
regions = {}
with httpx.Client(timeout=600) as client:
for example in json.loads((args.examples / "examples.json").read_text(encoding="utf-8")):
folder = args.examples / example["id"]
files = [(folder / f"line_{i:02d}.jpg").read_bytes() for i in range(len(example["lines"]))]
bodies = requests_for(folder / "region.png", args.runs + 1)
alone, draft = Vllm(args.alone, client), Vllm(args.draft, client)
times = RegionTimes(
lines=len(files),
trocr_single=median_seconds(lambda _, files=files: read_lines(files, 1), args.runs),
trocr_batch=median_seconds(lambda _, files=files: read_lines(files, len(files)), args.runs),
vllm=median_seconds(lambda run, alone=alone, bodies=bodies: alone.read(bodies[run]), args.runs),
vllm_dspark=median_seconds(lambda run, draft=draft, bodies=bodies: draft.read(bodies[run]), args.runs),
tokens=alone.tokens,
)
regions[example["id"]] = times
same = "the same text" if alone.texts == draft.texts and len(alone.texts) == 1 else "texts differ"
print(example["id"], times.model_dump(), f"{len(alone.texts)} and {len(draft.texts)} readings, {same}")
reference = Reference(gpu=args.gpu, measured=date.today().isoformat(), runs=args.runs, regions=regions)
args.out.write_text(reference.model_dump_json(indent=1) + "\n", encoding="utf-8")
if __name__ == "__main__":
main()
|