File size: 5,040 Bytes
c162d1f 757e891 c162d1f e994856 c162d1f e994856 c162d1f 757e891 c162d1f 757e891 c162d1f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 | from __future__ import annotations
import json
from pathlib import Path
from typing import Any, Dict, List, Optional
import pandas as pd
RADAR_METRIC_KEYS: Dict[str, List[str]] = {
"urbanvideo": ["recall_avg", "perception_avg", "reasoning_avg", "navigation_avg"],
"embodiednav": ["short_sr", "middle_sr", "long_sr"],
}
ENTRY_TYPE_CHOICES = ["Reference Baseline", "Organizer Evaluation"]
class DataLoader:
def __init__(self, benchmark_id: str, data_file: str = "./data/leaderboards.json"):
self.benchmark_id = benchmark_id
self.data_file = Path(data_file)
self.df_all: Optional[pd.DataFrame] = None
self.benchmark_config: Optional[dict[str, Any]] = None
self.metric_display_map: dict[str, str] = {}
self.display_to_internal_map: dict[str, str] = {}
self.lower_better: list[str] = []
self.dimension_metrics: list[str] = []
self.dimension_display_labels: list[str] = []
self.BASIC_METRICS: list[str] = []
self.DIMENSION_METRICS: list[str] = []
self.DIMENSION_MAP: dict[str, list[str]] = {}
self.ALL_METRICS: list[str] = []
self.METRIC_CHOICES: list[str] = []
def load_results(self) -> pd.DataFrame:
payload = json.loads(self.data_file.read_text(encoding="utf-8"))
benchmark = next(
(item for item in payload["benchmarks"] if item["id"] == self.benchmark_id),
None,
)
if benchmark is None:
raise ValueError(f"Benchmark '{self.benchmark_id}' not found in {self.data_file}")
self.benchmark_config = benchmark
self.metric_display_map = {
**benchmark["metricLabels"],
"entry_type": "Entry Type",
}
self.display_to_internal_map = {
display_name: internal_name
for internal_name, display_name in self.metric_display_map.items()
}
self.lower_better = benchmark.get("lowerBetter", [])
self.dimension_metrics = [
metric for metric in RADAR_METRIC_KEYS.get(self.benchmark_id, []) if metric in self.metric_display_map
]
self.dimension_display_labels = [
self.metric_display_map.get(metric, metric) for metric in self.dimension_metrics
]
self.DIMENSION_METRICS = list(self.dimension_metrics)
self.DIMENSION_MAP = {metric: [metric] for metric in self.dimension_metrics}
self.ALL_METRICS = self._flatten_metric_keys(benchmark)
self.METRIC_CHOICES = list(self.ALL_METRICS)
self.BASIC_METRICS = [metric for metric in self.ALL_METRICS if metric not in self.DIMENSION_METRICS]
rows: list[dict[str, Any]] = []
for row in benchmark["rows"]:
metrics = row.get("metrics", {})
flattened: dict[str, Any] = {
"Model": row["model"],
"entry_type": row.get("entryType", benchmark.get("defaultEntryType", "Reference Baseline")),
"note": row.get("note", ""),
}
for metric in self.ALL_METRICS:
flattened[metric] = metrics.get(metric)
rows.append(flattened)
df = pd.DataFrame(rows)
for metric in self.ALL_METRICS:
if metric in df.columns:
df[metric] = pd.to_numeric(df[metric], errors="coerce").round(2)
return df
def reload_data(self) -> str:
self.df_all = self.load_results()
return f"Loaded {len(self.df_all)} models for {self.benchmark_config['name']}"
def get_entry_type_choices(self) -> List[str]:
if self.df_all is None or "entry_type" not in self.df_all.columns:
return ["All"]
values = set(self.df_all["entry_type"].dropna())
return ["All"] + [choice for choice in ENTRY_TYPE_CHOICES if choice in values]
def get_metric_choices(self) -> List[str]:
return list(self.METRIC_CHOICES)
def get_display_metric_choices(self) -> List[str]:
return [self.metric_display_map.get(metric, metric) for metric in self.METRIC_CHOICES]
def to_internal_metric(self, display_metric: str) -> str:
return self.display_to_internal_map.get(display_metric, display_metric)
def get_metric_label(self, metric: str) -> str:
return self.metric_display_map.get(metric, metric)
def get_dimension_dataframe(self, displayed_models: List[str]) -> pd.DataFrame:
if self.df_all is None or not displayed_models or not self.dimension_metrics:
return pd.DataFrame()
columns = ["Model"] + [metric for metric in self.dimension_metrics if metric in self.df_all.columns]
df = self.df_all[self.df_all["Model"].isin(displayed_models)][columns].copy()
return df
def _flatten_metric_keys(self, benchmark: dict[str, Any]) -> List[str]:
ordered: list[str] = []
for group in benchmark["metricGroups"]:
for key in group["keys"]:
if key not in ordered:
ordered.append(key)
return ordered
|