""" qwen_test_runner.vision — image→JSON benchmark harness. Extends the text testbed to vision: a per-category task registry (mirroring the caption SLOT_REGISTRY), coordinate normalization, ground-truth metrics that replace substring grounding, a multi-model VLM runner over the Qwen3.5 / Qwen3-VL ladder, and an orchestrator that ranks models for the no-finetune labeler verdict. The data-driven modules (coords, model_registry, tasks_vision, metrics) are torch-free and import eagerly. The VLM runner and orchestrator are imported lazily so `import qwen_test_runner.vision` stays cheap on a CPU box. """ from __future__ import annotations from .coords import BBox, CoordSpace, to_canonical, from_canonical, detect_space, prompt_hint_for from .model_registry import ( MODEL_REGISTRY, ModelSpec, get_model, model_keys, models_that_fit, reasoning_variants, get_runner, ) from .tasks_vision import ( VISION_TASK_REGISTRY, VisionTaskSpec, get_task, category_names, pilot_categories, model_for, json_schema_for, gbnf_for, tool_schema_for, resolved_system_prompt, ) from .metrics import ( MetricResult, VisionRunMetrics, labeler_score, score_vision_sample, score_vision_run, ) def __getattr__(name: str): # StubVLMRunner + VLMResult are torch-free; VLMRunner pulls torch. if name in ("StubVLMRunner",): from .stub_runner import StubVLMRunner return StubVLMRunner if name == "VLMResult": from .runner_types import VLMResult return VLMResult if name == "VLMRunner": from .runners import VLMRunner return VLMRunner if name == "run_bench": from .bench import run_bench return run_bench raise AttributeError(f"module {__name__!r} has no attribute {name!r}") __all__ = [ # coords "BBox", "CoordSpace", "to_canonical", "from_canonical", "detect_space", "prompt_hint_for", # model registry "MODEL_REGISTRY", "ModelSpec", "get_model", "model_keys", "models_that_fit", "reasoning_variants", "get_runner", # tasks "VISION_TASK_REGISTRY", "VisionTaskSpec", "get_task", "category_names", "pilot_categories", "model_for", "json_schema_for", "gbnf_for", "tool_schema_for", "resolved_system_prompt", # metrics "MetricResult", "VisionRunMetrics", "labeler_score", "score_vision_sample", "score_vision_run", # lazy "VLMRunner", "StubVLMRunner", "run_bench", ]