Spaces:
Running on Zero
Running on Zero
File size: 2,413 Bytes
fed954e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 | """
qwen_test_runner.vision — image→JSON benchmark harness.
Extends the text testbed to vision: a per-category task registry (mirroring the
caption SLOT_REGISTRY), coordinate normalization, ground-truth metrics that
replace substring grounding, a multi-model VLM runner over the Qwen3.5 / Qwen3-VL
ladder, and an orchestrator that ranks models for the no-finetune labeler verdict.
The data-driven modules (coords, model_registry, tasks_vision, metrics) are
torch-free and import eagerly. The VLM runner and orchestrator are imported lazily
so `import qwen_test_runner.vision` stays cheap on a CPU box.
"""
from __future__ import annotations
from .coords import BBox, CoordSpace, to_canonical, from_canonical, detect_space, prompt_hint_for
from .model_registry import (
MODEL_REGISTRY, ModelSpec, get_model, model_keys, models_that_fit,
reasoning_variants, get_runner,
)
from .tasks_vision import (
VISION_TASK_REGISTRY, VisionTaskSpec, get_task, category_names, pilot_categories,
model_for, json_schema_for, gbnf_for, tool_schema_for, resolved_system_prompt,
)
from .metrics import (
MetricResult, VisionRunMetrics, labeler_score,
score_vision_sample, score_vision_run,
)
def __getattr__(name: str):
# StubVLMRunner + VLMResult are torch-free; VLMRunner pulls torch.
if name in ("StubVLMRunner",):
from .stub_runner import StubVLMRunner
return StubVLMRunner
if name == "VLMResult":
from .runner_types import VLMResult
return VLMResult
if name == "VLMRunner":
from .runners import VLMRunner
return VLMRunner
if name == "run_bench":
from .bench import run_bench
return run_bench
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = [
# coords
"BBox", "CoordSpace", "to_canonical", "from_canonical", "detect_space", "prompt_hint_for",
# model registry
"MODEL_REGISTRY", "ModelSpec", "get_model", "model_keys", "models_that_fit",
"reasoning_variants", "get_runner",
# tasks
"VISION_TASK_REGISTRY", "VisionTaskSpec", "get_task", "category_names", "pilot_categories",
"model_for", "json_schema_for", "gbnf_for", "tool_schema_for", "resolved_system_prompt",
# metrics
"MetricResult", "VisionRunMetrics", "labeler_score",
"score_vision_sample", "score_vision_run",
# lazy
"VLMRunner", "StubVLMRunner", "run_bench",
]
|