"""Centralized constants for the VANTAGE-Bench leaderboard. All ordering, naming, and bucket definitions live here so that downstream modules (data, ranking, render) read from a single source of truth. """ from __future__ import annotations # -- Version / metadata ---------------------------------------------------- VERSION = "v1.0" UPDATED = "2026-05-23" # -- Schema ----------------------------------------------------------------- SCHEMA_VERSION = "1.0" # -- Pillars ---------------------------------------------------------------- # Maps pillar key → ordered list of short task keys belonging to that pillar. # 'overall' contains every task (used for averaging across all pillars). # Short task keys match the TASKS dict below. PILLARS: dict[str, list[str]] = { "overall": ["loc", "ground", "pointing", "sot", "temploc", "dvc", "ev", "vqa"], "spatial": ["loc", "ground", "pointing"], "st": ["sot"], "temporal": ["temploc", "dvc"], "semantic": ["ev", "vqa"], } # -- Tasks ------------------------------------------------------------------ # Short task key → display label shown in column headers and side-panel. TASKS: dict[str, str] = { "loc": "2D Object Localization", "ground": "2D Referring Expressions", "pointing": "2D Pointing", "sot": "Single Object Tracking", "temploc": "Temporal Localization", "dvc": "Dense Video Captioning", "ev": "Event Verification", "vqa": "Video QA", } # -- Parameter buckets ------------------------------------------------------ # Short bucket key → display label for the filter UI. # Matching logic lives in data.py (_param_bucket helper). PARAM_BUCKETS: dict[str, str] = { "all": "All sizes", "lt10": "<10B", "10to40": "10B–40B", "gt40": ">40B", } # -- Badge colors ----------------------------------------------------------- # badge type → (background_hex, text_hex, border_hex) BADGE_COLORS: dict[str, tuple[str, str, str]] = { "verified": ("#dcfce7", "#15803d", "#86efac"), "ensemble": ("#dbeafe", "#1d4ed8", "#93c5fd"), "single": ("#f3f4f6", "#374151", "#d1d5db"), "open": ("#ccfbf1", "#0f766e", "#5eead4"), "proprietary": ("#fef3c7", "#92400e", "#fcd34d"), "new": ("#ede9fe", "#5b21b6", "#c4b5fd"), } # ── Everything below this line is preserved for compatibility with existing # modules (data.py, ranking.py, render.py, filters.py, app.py). # Do not remove or rename these without updating the importing modules. # -------------------------------------------------------------------------- # -- Canonical pillar + tab order ------------------------------------------ LEADERBOARD_TABS: list[str] = [ "overall", "spatial", "spatio_temporal", "temporal", "semantic", ] HEADLINE_FIELD_BY_TAB: dict[str, str] = { "overall": "overall", "spatial": "spatial", "spatio_temporal": "spatio_temporal", "temporal": "temporal", "semantic": "semantic", } # -- Required / optional field sets ---------------------------------------- REQUIRED_MODEL_FIELDS: list[str] = [ "id", "name", "organization", "params", "type", "result_type", ] REQUIRED_SCORE_FIELDS: list[str] = [ "overall", "spatial", "spatio_temporal", "temporal", "semantic", ] OPTIONAL_TASK_FIELDS: list[str] = [ "2d_localization", "2d_referring_expressions", "2d_spatial_pointing", "single_object_tracking", "temporal_localization", "dense_video_captioning", "event_verification", "video_qa", ] ALL_SCORE_FIELDS: list[str] = REQUIRED_SCORE_FIELDS + OPTIONAL_TASK_FIELDS OPTIONAL_METADATA_FIELDS: list[str] = ["url"] ALLOWED_TYPES: tuple[str, ...] = ("open", "closed") ALLOWED_RESULT_TYPES: tuple[str, ...] = ("single", "ensemble") TYPE_DISPLAY: dict[str, str] = { "open": "open", "closed": "proprietary", } # -- Parameter buckets (legacy list format used by app.py dropdowns) -------- _PARAM_BUCKET_DEFS: list[tuple[str, float | None, float | None]] = [ ("All", None, None), ("<3B", 0.0, 3.0), ("3–10B", 3.0, 10.0), ("10–30B", 10.0, 30.0), ("30B+", 30.0, None), ] PARAM_BUCKET_LABELS: list[str] = [b[0] for b in _PARAM_BUCKET_DEFS] # -- Model type filter ----------------------------------------------------- MODEL_TYPE_LABELS: list[str] = ["All", "Single", "System / Pipeline"] # UI labels → internal enum value stored in ModelRecord.result_type. # "System / Pipeline" is the user-facing label; "ensemble" remains the # internal value for schema/data backwards compatibility. MODEL_TYPE_TO_INTERNAL: dict[str, str | None] = { "All": None, "Single": "single", "System / Pipeline": "ensemble", } # -- Display labels -------------------------------------------------------- PILLAR_LABELS: dict[str, str] = { "spatial": "Spatial", "spatio_temporal": "Spatio-Temporal", "temporal": "Temporal", "semantic": "Semantic", } PILLAR_LABELS_SHORT: dict[str, str] = { "spatial": "Spatial", "spatio_temporal": "Sp-Temp", "temporal": "Temporal", "semantic": "Semantic", } TAB_LABELS: dict[str, str] = { "overall": "Overall", "spatial": "Spatial", "spatio_temporal": "Spatio-Temporal", "temporal": "Temporal", "semantic": "Semantic", } TASK_LABELS: dict[str, str] = { "2d_localization": "Object Localization", "2d_referring_expressions": "Referring Expressions", "2d_spatial_pointing": "2D Pointing", "single_object_tracking": "Single Object Tracking", "temporal_localization": "Temporal Localization", "dense_video_captioning": "Dense Video Captioning", "event_verification": "Event Verification", "video_qa": "Video Question Answering", } TASK_LABELS_SHORT: dict[str, str] = { "2d_localization": "Obj.Loc.", "2d_referring_expressions": "Ground", "2d_spatial_pointing": "Point", "single_object_tracking": "SOT", "temporal_localization": "Temp-Loc", "dense_video_captioning": "DVC", "event_verification": "EV", "video_qa": "VQA", } TASK_METRIC_LABELS: dict[str, str] = { "2d_localization": "F1@0.5", "2d_referring_expressions": "mIoU", "2d_spatial_pointing": "Accuracy", "single_object_tracking": "AUC", "temporal_localization": "mIoU", "dense_video_captioning": "SODAc", "event_verification": "Macro F1", "video_qa": "Accuracy", } TASKS_BY_PILLAR: dict[str, list[str]] = { "spatial": [ "2d_localization", "2d_referring_expressions", "2d_spatial_pointing", ], "spatio_temporal": [ "single_object_tracking", ], "temporal": [ "temporal_localization", "dense_video_captioning", ], "semantic": [ "event_verification", "video_qa", ], }