evidence/scripts/benchmark_foundation_http.py

"""Measure real TCP requests with distinct held-out photographs and resident weights."""

import argparse
import hashlib
import json
import socket
import threading
import time
from pathlib import Path

import httpx
import numpy as np
import torch
import uvicorn
from PIL import Image
from train_foundation_head import write
from validate_model import latency_summary

from veyra.candidates import candidates_for
from veyra.data import read_records
from veyra.model import VeyraModel
from veyra.server import create_app


def digest(path):
    return hashlib.sha256(path.read_bytes()).hexdigest()


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--checkpoint", type=Path, required=True)
    parser.add_argument("--records", type=Path, required=True)
    parser.add_argument("--output", type=Path, required=True)
    parser.add_argument("--iterations", type=int, default=40)
    args = parser.parse_args()
    if args.output.exists() or args.iterations < 20:
        raise ValueError("use a new output and at least 20 measured photographs")
    pool = [
        record
        for record in read_records(args.records)
        if record.split == "test"
        and "domain:image_waste" in record.tags
        and "view:recognition" in record.tags
    ]
    groups = {}
    for record in sorted(pool, key=lambda r: hashlib.sha256(r.id.encode()).hexdigest()):
        groups.setdefault(record.group_id, record)
    chosen = list(groups.values())[: args.iterations + 3]
    if len(chosen) != args.iterations + 3:
        raise ValueError("insufficient distinct photograph groups")
    if any(
        len(r.request.state.images) != 1
        or len(r.request.questions) != 1
        or len(candidates_for(next(iter(r.request.questions.values())))) != 6
        for r in chosen
    ):
        raise ValueError("benchmark requires one photo, one question and six candidates")
    root = args.records.parent.resolve()
    dimensions = []
    for record in chosen:
        with Image.open(root / record.request.state.images[0].path) as image:
            dimensions.append(list(image.size))
    protocol = {
        "weights_sha256": digest(args.checkpoint / "head.safetensors"),
        "manifest_sha256": digest(args.checkpoint / "manifest.json"),
        "dataset_sha256": digest(args.records),
        "script_sha256": digest(Path(__file__)),
        "record_ids": [r.id for r in chosen],
        "image_dimensions": dimensions,
        "scope": (
            "One real photograph, one recognition question, six candidates; "
            "local HTTP round trip including JSON and image decode"
        ),
        "warmups": 3,
        "photos_per_request": 1,
        "questions_per_request": 1,
        "candidates_per_question": 6,
        "resident": True,
        "feature_cache": False,
        "serial": True,
        "excludes": ["model loading", "WAN transport", "concurrent load"],
        "selection": "Fixed ID hash ordering, unique image groups; no correctness filtering",
    }
    write(args.output.with_suffix(".protocol.json"), protocol)
    torch.set_num_threads(4)
    started = time.perf_counter()
    model = VeyraModel.load(args.checkpoint, local_files_only=True)
    load_seconds = time.perf_counter() - started
    torch.cuda.reset_peak_memory_stats()
    sock = socket.socket()
    sock.bind(("127.0.0.1", 0))
    port = sock.getsockname()[1]
    server = uvicorn.Server(uvicorn.Config(create_app(model, root), log_level="warning"))
    thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True)
    thread.start()
    timings, tokens = [], []
    try:
        deadline = time.monotonic() + 15
        while not server.started and time.monotonic() < deadline:
            time.sleep(0.05)
        if not server.started:
            raise RuntimeError("local server did not become ready")
        with httpx.Client(
            base_url=f"http://127.0.0.1:{port}", timeout=30, trust_env=False
        ) as client:
            for index, record in enumerate(chosen):
                torch.cuda.synchronize()
                start = time.perf_counter()
                response = client.post("/v1/systemone", json=record.request.model_dump())
                response.raise_for_status()
                result = response.json()
                elapsed = 1000 * (time.perf_counter() - start)
                if result["usage"]["output_tokens"] != 0:
                    raise ValueError("unexpected generated tokens")
                if index >= 3:
                    timings.append(elapsed)
                    tokens.append(result["usage"]["input_tokens"])
    finally:
        server.should_exit = True
        thread.join(timeout=15)
        sock.close()
    result = {
        "protocol": protocol,
        "gpu": torch.cuda.get_device_name(),
        "gpu_memory_mib": torch.cuda.get_device_properties(0).total_memory / 2**20,
        "load_seconds": load_seconds,
        "requests": len(timings),
        "latency_ms": latency_summary(timings),
        "p95_target_met": bool(np.percentile(timings, 95) <= 300),
        "input_token_range": [min(tokens), max(tokens)],
        "peak_torch_allocated_mib": torch.cuda.max_memory_allocated() / 2**20,
    }
    write(args.output, result)
    print(
        json.dumps(
            {
                "requests": len(timings),
                "p50": result["latency_ms"]["p50"],
                "p95": result["latency_ms"]["p95"],
                "target_met": result["p95_target_met"],
            }
        ),
        flush=True,
    )


if __name__ == "__main__":
    main()