from __future__ import annotations import html import math from dataclasses import dataclass from pathlib import Path from typing import Iterable import pandas as pd DATA_PATH = Path(__file__).parent / "data" / "leaderboard.csv" METRIC_COLUMNS = [ "Vis", "Aud (PQ)", "AV", "Lip", "Text", "Face", "Music", "Speech", "Lo-Phy", "Hi-Phy", "Holistic", "Total", ] SORT_COLUMNS = ["Rank", "Model", "Components", "Component Type", *METRIC_COLUMNS] SORT_CHOICES = [ ("Rank", "Rank"), ("Model", "Model"), ("Components", "Components"), ("Type", "Component Type"), *[(metric, metric) for metric in METRIC_COLUMNS], ] LOWER_IS_BETTER = {"AV", "Lip"} NUMERIC_COLUMNS = METRIC_COLUMNS FORMATTERS = { "Vis": "{:.3f}", "Aud (PQ)": "{:.2f}", "AV": "{:.2f}", "Lip": "{:.2f}", "Text": "{:.2f}", "Face": "{:.2f}", "Music": "{:.2f}", "Speech": "{:.2f}", "Lo-Phy": "{:.2f}", "Hi-Phy": "{:.2f}", "Holistic": "{:.2f}", "Total": "{:.2f}", } GROUP_WEIGHTS = { "Basic Uni-modal": 0.2, "Basic Cross-modal": 0.2, "Fine-grained": 0.6, } GROUP_DIMENSIONS = { "Basic Uni-modal": ["Vis", "Aud (PQ)"], "Basic Cross-modal": ["AV", "Lip"], "Fine-grained": ["Text", "Face", "Music", "Speech", "Lo-Phy", "Hi-Phy", "Holistic"], } @dataclass(frozen=True) class Standing: best: float | None second: float | None def load_leaderboard(path: Path = DATA_PATH) -> pd.DataFrame: df = pd.read_csv(path) for column in NUMERIC_COLUMNS: df[column] = pd.to_numeric(df[column], errors="coerce") df = df.sort_values("Total", ascending=False, na_position="last").reset_index(drop=True) df.insert(0, "Rank", range(1, len(df) + 1)) return df def filter_leaderboard( df: pd.DataFrame, component_type: str = "All", query: str = "", sort_by: str = "Total", sort_order: str = "Descending", ) -> pd.DataFrame: view = df.copy() if component_type != "All": view = view[view["Component Type"] == component_type] query = query.strip().lower() if query: mask = ( view["Model"].str.lower().str.contains(query, regex=False) | view["Components"].str.lower().str.contains(query, regex=False) ) view = view[mask] sort_by = _normalize_sort_column(sort_by) ascending = _is_ascending_sort(sort_by, sort_order) if sort_by in view.columns: sort_kwargs = { "ascending": ascending, "na_position": "last", "kind": "mergesort", } if sort_by in {"Model", "Components", "Component Type"}: sort_kwargs["key"] = lambda column: column.astype(str).str.casefold() view = view.sort_values(sort_by, **sort_kwargs) return view.reset_index(drop=True) def metric_standings(df: pd.DataFrame) -> dict[str, Standing]: standings: dict[str, Standing] = {} for metric in METRIC_COLUMNS: values = sorted( {float(v) for v in df[metric].dropna()}, reverse=metric not in LOWER_IS_BETTER, ) standings[metric] = Standing( best=values[0] if values else None, second=values[1] if len(values) > 1 else None, ) return standings def normalized_score(metric: str, value: float) -> float: if metric == "Vis": return _clamp(value * 100.0, 0.0, 100.0) if metric == "Aud (PQ)": return _clamp(value * 10.0, 0.0, 100.0) if metric == "AV": return _clamp(100.0 * (1.0 - value / 0.5), 0.0, 100.0) if metric == "Lip": return _clamp(100.0 * (1.0 - value / 8.0), 0.0, 100.0) if metric == "Lo-Phy": return _clamp(value * 20.0, 0.0, 100.0) return _clamp(value, 0.0, 100.0) def compute_total_from_metrics(row: pd.Series) -> float: group_scores: list[float] = [] group_weights: list[float] = [] for group_name, metrics in GROUP_DIMENSIONS.items(): values = [] for metric in metrics: value = row.get(metric) if pd.isna(value): continue values.append(normalized_score(metric, float(value))) if values: group_scores.append(sum(values) / len(values)) group_weights.append(GROUP_WEIGHTS[group_name]) if not group_scores: return float("nan") weighted = sum(score * weight for score, weight in zip(group_scores, group_weights)) return weighted / sum(group_weights) def render_summary(df: pd.DataFrame, view: pd.DataFrame) -> str: top = df.sort_values("Total", ascending=False).iloc[0] open_source = df[df["Component Type"] == "Open-source"].sort_values("Total", ascending=False) best_open = open_source.iloc[0] if len(open_source) else None best_av = df.sort_values("AV", ascending=True).iloc[0] best_speech = df.sort_values("Speech", ascending=False).iloc[0] cards = [ _summary_card("Models", f"{len(view)} / {len(df)}", "shown in current view"), _summary_card("Top Total", _score(top["Total"]), str(top["Model"])), _summary_card( "Best Open-source", _score(best_open["Total"]) if best_open is not None else "NA", str(best_open["Model"]) if best_open is not None else "No entry", ), _summary_card("Lowest AV Offset", _score(best_av["AV"]), str(best_av["Model"])), _summary_card("Highest Speech", _score(best_speech["Speech"]), str(best_speech["Model"])), ] return '
Model profile
{html.escape(', '.join(metrics))}
Total uses AVGen-Bench Scheme 2: group-weighted normalized metrics with Vis x 100, Aud(PQ) x 10, Lo-Phy x 20, AV = 100 * max(0, 1 - AV / 0.5), Lip = 100 * max(0, 1 - Lip / 8), and the remaining metrics already on a 0-100 scale.