File size: 6,962 Bytes
b0dacaa
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
"""The Space's reference timings: each example region read on this machine by the Swedish Lion TrOCR (transformers,
its lines one after the other and as one batch, as the app runs it) and by the reader in vLLM, alone and with its
DSpark draft, one request at a time. The app shows them beside its live race, which runs in PyTorch because ZeroGPU
cannot host vLLM.

Serve the reader twice first, as the draft's card says (prefix caching off), the second server once the first
answers, on a GPU nothing else is using:

    vllm serve READER --served-model-name htr ... --port 8057
    vllm serve READER --served-model-name htr ... --port 8056 --speculative-config '{"method": "dspark", ...}'
    python measure_reference.py --examples examples --gpu "RTX PRO 6000 Blackwell" --out reference.json

Each reading is timed from the encoded images to the reading: the median of RUNS runs after one that is not timed.
vLLM keeps the processed form of an image it has seen, so every request sends the region with one pixel, drawn at
random, changed by one level: the region is new to the server each time, as a region is in production.
"""

from __future__ import annotations

import argparse
import base64
import io
import json
import secrets
import statistics
import time
from collections.abc import Callable
from datetime import date
from pathlib import Path
from typing import Any

import httpx
import torch
from PIL import Image
from transformers import TrOCRProcessor, VisionEncoderDecoderModel

from reference import Reference, RegionTimes

SAMPLING = {"temperature": 0.01, "top_p": 0.001, "top_k": 1, "repetition_penalty": 1.0}  # OpenBMB's, near greedy
IMAGES = {"downsample_mode": "4x", "max_slice_nums": 9}


def median_seconds(read: Callable[[int], None], runs: int) -> float:
    """The median time of `runs` calls of `read`, after one that is not timed; each call gets its own number."""
    read(0)
    times = []
    for run in range(1, runs + 1):
        started = time.perf_counter()
        read(run)
        times.append(time.perf_counter() - started)
    return round(statistics.median(times), 3)


def requests_for(region: Path, count: int) -> list[dict[str, Any]]:
    """`count` requests for one region, each with one pixel, drawn at random, changed by one level."""
    image = Image.open(region).convert("RGB")
    pixels = image.tobytes()
    bodies = []
    for _ in range(count):
        changed = bytearray(pixels)
        changed[3 * secrets.randbelow(image.width * image.height)] ^= 1  # a pixel's red
        buffer = io.BytesIO()
        Image.frombytes("RGB", image.size, bytes(changed)).save(buffer, format="PNG")
        url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
        content = [{"type": "image_url", "image_url": {"url": url}}, {"type": "text", "text": "Text Recognition:"}]
        bodies.append(
            {
                "model": "htr",
                **SAMPLING,
                "max_tokens": 4096,
                "chat_template_kwargs": {"enable_thinking": False},
                "mm_processor_kwargs": IMAGES,
                "messages": [{"role": "user", "content": content}],
            }
        )
    return bodies


class Vllm:
    """One vLLM server, asked for one region at a time; it keeps what the region's requests answered."""

    def __init__(self, endpoint: str, client: httpx.Client) -> None:
        self.url = f"{endpoint}/v1/chat/completions"
        self.client = client
        self.texts: set[str] = set()
        self.tokens = 0

    def read(self, body: dict[str, Any]) -> None:
        reply = self.client.post(self.url, json=body)
        reply.raise_for_status()
        answer = reply.json()
        self.texts.add(answer["choices"][0]["message"]["content"])
        self.tokens = answer["usage"]["completion_tokens"]


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--examples", type=Path, required=True, help="the folder build_examples.py wrote")
    parser.add_argument("--trocr", default="Gabriel/swedish-lion-trocr-large-handwritten-char")
    parser.add_argument("--alone", default="http://localhost:8057", help="vLLM serving the reader alone")
    parser.add_argument("--draft", default="http://localhost:8056", help="vLLM serving the reader with its draft")
    parser.add_argument("--gpu", required=True, help="the GPU's name, as the app will show it")
    parser.add_argument("--runs", type=int, default=5)
    parser.add_argument("--out", type=Path, required=True)
    args = parser.parse_args()

    processor = TrOCRProcessor.from_pretrained(args.trocr)
    trocr = VisionEncoderDecoderModel.from_pretrained(args.trocr).to("cuda").eval()

    def read_lines(files: list[bytes], batch: int) -> None:
        lines = [Image.open(io.BytesIO(data)).convert("RGB") for data in files]
        for first in range(0, len(lines), batch):
            pixels = processor(images=lines[first : first + batch], return_tensors="pt").pixel_values.to("cuda")
            with torch.inference_mode():
                trocr.generate(
                    pixels,
                    decoder_start_token_id=trocr.config.decoder_start_token_id,
                    interpolate_pos_encoding=True,
                    num_beams=1,
                    max_new_tokens=128,
                )
        torch.cuda.synchronize()

    regions = {}
    with httpx.Client(timeout=600) as client:
        for example in json.loads((args.examples / "examples.json").read_text(encoding="utf-8")):
            folder = args.examples / example["id"]
            files = [(folder / f"line_{i:02d}.jpg").read_bytes() for i in range(len(example["lines"]))]
            bodies = requests_for(folder / "region.png", args.runs + 1)
            alone, draft = Vllm(args.alone, client), Vllm(args.draft, client)
            times = RegionTimes(
                lines=len(files),
                trocr_single=median_seconds(lambda _, files=files: read_lines(files, 1), args.runs),
                trocr_batch=median_seconds(lambda _, files=files: read_lines(files, len(files)), args.runs),
                vllm=median_seconds(lambda run, alone=alone, bodies=bodies: alone.read(bodies[run]), args.runs),
                vllm_dspark=median_seconds(lambda run, draft=draft, bodies=bodies: draft.read(bodies[run]), args.runs),
                tokens=alone.tokens,
            )
            regions[example["id"]] = times
            same = "the same text" if alone.texts == draft.texts and len(alone.texts) == 1 else "texts differ"
            print(example["id"], times.model_dump(), f"{len(alone.texts)} and {len(draft.texts)} readings, {same}")
    reference = Reference(gpu=args.gpu, measured=date.today().isoformat(), runs=args.runs, regions=regions)
    args.out.write_text(reference.model_dump_json(indent=1) + "\n", encoding="utf-8")


if __name__ == "__main__":
    main()