Spaces:
Running
Running
Deploy Space: update src
Browse files- src/utils.py +54 -12
src/utils.py
CHANGED
|
@@ -59,6 +59,7 @@ RANK_SCORE_BASE_METRICS = [
|
|
| 59 |
"eval_metrics/MSE[mean]",
|
| 60 |
"eval_metrics/mean_weighted_sum_quantile_loss",
|
| 61 |
]
|
|
|
|
| 62 |
|
| 63 |
VALUE_COLUMNS = ["MSE", "CRPS", RANK_SCORE_COLUMN] + [
|
| 64 |
METRIC_LABELS[m] for m in DISPLAY_METRICS if METRIC_LABELS[m] not in {"MSE", "CRPS"}
|
|
@@ -163,16 +164,23 @@ LIVE_AGGREGATE_FILES = {
|
|
| 163 |
|
| 164 |
|
| 165 |
def format_number(value):
|
| 166 |
-
"""Format a displayed leaderboard metric
|
| 167 |
if pd.isna(value):
|
| 168 |
-
return
|
| 169 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
|
| 171 |
|
| 172 |
def aggregate_gmean(series: pd.Series) -> float:
|
| 173 |
-
values = series.dropna()
|
|
|
|
| 174 |
if values.empty:
|
| 175 |
return np.nan
|
|
|
|
| 176 |
return float(stats.gmean(values))
|
| 177 |
|
| 178 |
|
|
@@ -262,6 +270,7 @@ def prepare_results_df(
|
|
| 262 |
|
| 263 |
for metric in METRIC_COLUMNS:
|
| 264 |
df[metric] = pd.to_numeric(df[metric], errors="coerce")
|
|
|
|
| 265 |
|
| 266 |
return _add_per_dataset_ranks(df)
|
| 267 |
|
|
@@ -652,6 +661,46 @@ def group_datasets_by_domain(
|
|
| 652 |
return ordered
|
| 653 |
|
| 654 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 655 |
def build_leaderboard_summary_html(root_dir: str = "results") -> str:
|
| 656 |
df = prepare_results_df(root_dir)
|
| 657 |
if df.empty:
|
|
@@ -667,14 +716,7 @@ def build_leaderboard_summary_html(root_dir: str = "results") -> str:
|
|
| 667 |
models = df["model"].nunique()
|
| 668 |
datasets = df["dataset"].nunique()
|
| 669 |
domains = df["domain"].nunique() if "domain" in df.columns else 0
|
| 670 |
-
last_refresh =
|
| 671 |
-
status_path = Path(root_dir) / "online_status.json"
|
| 672 |
-
if status_path.exists():
|
| 673 |
-
try:
|
| 674 |
-
status = json.loads(status_path.read_text())
|
| 675 |
-
last_refresh = format_timestamp_utc8(status.get("finished_at", ""))
|
| 676 |
-
except Exception:
|
| 677 |
-
pass
|
| 678 |
|
| 679 |
return f"""
|
| 680 |
<div class="summary-grid">
|
|
|
|
| 59 |
"eval_metrics/MSE[mean]",
|
| 60 |
"eval_metrics/mean_weighted_sum_quantile_loss",
|
| 61 |
]
|
| 62 |
+
GEOMEAN_EPSILON = 1e-12
|
| 63 |
|
| 64 |
VALUE_COLUMNS = ["MSE", "CRPS", RANK_SCORE_COLUMN] + [
|
| 65 |
METRIC_LABELS[m] for m in DISPLAY_METRICS if METRIC_LABELS[m] not in {"MSE", "CRPS"}
|
|
|
|
| 164 |
|
| 165 |
|
| 166 |
def format_number(value):
|
| 167 |
+
"""Format a displayed leaderboard metric without hiding tiny non-zero values."""
|
| 168 |
if pd.isna(value):
|
| 169 |
+
return "n/a"
|
| 170 |
+
number = float(value)
|
| 171 |
+
if not np.isfinite(number):
|
| 172 |
+
return "n/a"
|
| 173 |
+
if number != 0.0 and abs(number) < 0.005:
|
| 174 |
+
return f"{number:.2e}"
|
| 175 |
+
return f"{number:.2f}"
|
| 176 |
|
| 177 |
|
| 178 |
def aggregate_gmean(series: pd.Series) -> float:
|
| 179 |
+
values = pd.to_numeric(series, errors="coerce").dropna()
|
| 180 |
+
values = values[np.isfinite(values) & (values >= 0.0)]
|
| 181 |
if values.empty:
|
| 182 |
return np.nan
|
| 183 |
+
values = values.clip(lower=GEOMEAN_EPSILON)
|
| 184 |
return float(stats.gmean(values))
|
| 185 |
|
| 186 |
|
|
|
|
| 270 |
|
| 271 |
for metric in METRIC_COLUMNS:
|
| 272 |
df[metric] = pd.to_numeric(df[metric], errors="coerce")
|
| 273 |
+
df.loc[~np.isfinite(df[metric]), metric] = np.nan
|
| 274 |
|
| 275 |
return _add_per_dataset_ranks(df)
|
| 276 |
|
|
|
|
| 661 |
return ordered
|
| 662 |
|
| 663 |
|
| 664 |
+
def _parse_timestamp(value: object) -> datetime | None:
|
| 665 |
+
if not value or value == "n/a":
|
| 666 |
+
return None
|
| 667 |
+
try:
|
| 668 |
+
return datetime.fromisoformat(str(value).replace("Z", "+00:00"))
|
| 669 |
+
except (TypeError, ValueError):
|
| 670 |
+
return None
|
| 671 |
+
|
| 672 |
+
|
| 673 |
+
def _read_json(path: Path) -> dict:
|
| 674 |
+
if not path.exists():
|
| 675 |
+
return {}
|
| 676 |
+
try:
|
| 677 |
+
return json.loads(path.read_text())
|
| 678 |
+
except (OSError, json.JSONDecodeError):
|
| 679 |
+
return {}
|
| 680 |
+
|
| 681 |
+
|
| 682 |
+
def leaderboard_refresh_timestamp(root_dir: str = "results") -> str:
|
| 683 |
+
"""Return the latest timestamp that can change leaderboard-visible results."""
|
| 684 |
+
root = Path(root_dir)
|
| 685 |
+
candidates: list[datetime] = []
|
| 686 |
+
|
| 687 |
+
status = _read_json(root / "online_status.json")
|
| 688 |
+
for key in ("finished_at", "pushed_at"):
|
| 689 |
+
parsed = _parse_timestamp(status.get(key))
|
| 690 |
+
if parsed is not None:
|
| 691 |
+
candidates.append(parsed)
|
| 692 |
+
|
| 693 |
+
for metadata_name in ("metadata.json", "live_metadata.json"):
|
| 694 |
+
metadata = _read_json(root / "aggregates" / metadata_name)
|
| 695 |
+
parsed = _parse_timestamp(metadata.get("generated_at"))
|
| 696 |
+
if parsed is not None:
|
| 697 |
+
candidates.append(parsed)
|
| 698 |
+
|
| 699 |
+
if not candidates:
|
| 700 |
+
return "n/a"
|
| 701 |
+
return format_timestamp_utc8(max(candidates).isoformat())
|
| 702 |
+
|
| 703 |
+
|
| 704 |
def build_leaderboard_summary_html(root_dir: str = "results") -> str:
|
| 705 |
df = prepare_results_df(root_dir)
|
| 706 |
if df.empty:
|
|
|
|
| 716 |
models = df["model"].nunique()
|
| 717 |
datasets = df["dataset"].nunique()
|
| 718 |
domains = df["domain"].nunique() if "domain" in df.columns else 0
|
| 719 |
+
last_refresh = leaderboard_refresh_timestamp(root_dir)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 720 |
|
| 721 |
return f"""
|
| 722 |
<div class="summary-grid">
|