Deploy Space: update src

src/about.py

@@ -5,8 +5,9 @@ INTRODUCTION_TEXT = """
 real-world data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
 Following [GIFT-Eval](https://huggingface.co/spaces/Salesforce/GIFT-Eval), the leaderboard
 reports **absolute metric values** (not normalized to a baseline model) and **per-dataset
-ranks**. Use the **Overall** tab for aggregate scores; each subsequent tab is one **dataset
-(domain)** with its own absolute results.
+ranks**. The Overall tab also reports **RankScore**, an Elo-style rank-based aggregate
+computed from per-dataset MSE and CRPS ranks. Use the **Overall** tab for aggregate scores;
+each subsequent tab is one **dataset (domain)** with its own absolute results.
 """
 
 LLM_BENCHMARKS_TEXT = """
@@ -36,9 +37,10 @@ Once your Pull Request is opened, our automated sandbox pipeline will load your
 
 ## Metrics
 
-- **MASE** — Mean Absolute Scaled Error (absolute)
+- **MSE** — Mean Squared Error on the mean forecast (absolute)
 - **CRPS** — Mean Weighted Sum Quantile Loss (probabilistic forecast quality)
-- **MASE_Rank** / **CRPS_Rank** — per-dataset rank (lower is better)
+- **RankScore** — Elo-style aggregate from per-dataset MSE and CRPS ranks (higher is better)
+- **MSE_Rank** / **CRPS_Rank** — per-dataset rank (lower is better)
 """
 
 CITATION_BUTTON_LABEL = "Copy citation"

src/rank_based_metric.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from typing import Dict, Sequence, Tuple
+
+import numpy as np
+import pandas as pd
+
+
+def expected_score(rating_a: float, rating_b: float) -> float:
+    return 1.0 / (1.0 + 10 ** ((rating_b - rating_a) / 400.0))
+
+
+def update_ratings(rating_a: float, rating_b: float, score_a: float, k_factor: float) -> Tuple[float, float]:
+    expected_a = expected_score(rating_a, rating_b)
+    expected_b = expected_score(rating_b, rating_a)
+    score_b = 1.0 - score_a
+    return (
+        rating_a + k_factor * (score_a - expected_a),
+        rating_b + k_factor * (score_b - expected_b),
+    )
+
+
+def calculate_elo(
+    df_long: pd.DataFrame,
+    initial_rating: int = 1000,
+    k_factor: float = 32,
+    tie_epsilon: float = 0.0,
+) -> Dict[str, float]:
+    required = {"dataset", "model", "score"}
+    missing = required - set(df_long.columns)
+    if missing:
+        raise ValueError(f"df_long is missing columns: {sorted(missing)}")
+
+    clean = df_long.loc[:, ["dataset", "model", "score"]].copy()
+    clean["score"] = pd.to_numeric(clean["score"], errors="coerce")
+    clean = clean.dropna(subset=["dataset", "model"])
+
+    models = sorted(clean["model"].astype(str).unique())
+    ratings = {model: float(initial_rating) for model in models}
+
+    for dataset in sorted(clean["dataset"].astype(str).unique()):
+        sub = clean[clean["dataset"].astype(str) == dataset].dropna(subset=["score"])
+        scores = sub.groupby("model")["score"].mean()
+        present_models = sorted(str(model) for model in scores.index)
+        for idx, model_a in enumerate(present_models):
+            for model_b in present_models[idx + 1 :]:
+                score_a_value = float(scores.loc[model_a])
+                score_b_value = float(scores.loc[model_b])
+                if score_a_value > score_b_value + tie_epsilon:
+                    match_score_a = 1.0
+                elif score_b_value > score_a_value + tie_epsilon:
+                    match_score_a = 0.0
+                else:
+                    match_score_a = 0.5
+
+                rating_a, rating_b = update_ratings(
+                    ratings[model_a],
+                    ratings[model_b],
+                    match_score_a,
+                    k_factor,
+                )
+                ratings[model_a] = rating_a
+                ratings[model_b] = rating_b
+
+    return ratings
+
+
+def compute_rank_score(
+    df: pd.DataFrame,
+    metric_columns: Sequence[str],
+    group_columns: Sequence[str] = ("dataset",),
+    *,
+    initial_rating: int = 1000,
+    k_factor: float = 32,
+) -> pd.Series:
+    required = {"model", *group_columns, *metric_columns}
+    missing = required - set(df.columns)
+    if missing:
+        raise ValueError(f"df is missing columns: {sorted(missing)}")
+
+    work = df.loc[:, ["model", *group_columns, *metric_columns]].copy()
+    for metric in metric_columns:
+        work[metric] = pd.to_numeric(work[metric], errors="coerce")
+        work[f"__rank_{metric}"] = work.groupby(list(group_columns))[metric].rank(
+            method="average",
+            ascending=True,
+        )
+
+    rank_cols = [f"__rank_{metric}" for metric in metric_columns]
+    work["score"] = -work[rank_cols].mean(axis=1, skipna=True)
+    work = work.dropna(subset=["score"])
+    work["dataset"] = work.loc[:, list(group_columns)].astype(str).agg("::".join, axis=1)
+
+    ratings = calculate_elo(
+        work.loc[:, ["dataset", "model", "score"]],
+        initial_rating=initial_rating,
+        k_factor=k_factor,
+    )
+    return pd.Series(ratings, name="RankScore")

src/utils.py

@@ -11,6 +11,7 @@ import pandas as pd
 from scipy import stats
 
 from src.display.formatting import format_timestamp_utc8
+from src.rank_based_metric import compute_rank_score
 
 METRIC_COLUMNS = [
     "eval_metrics/MSE[mean]",
@@ -27,8 +28,9 @@ METRIC_COLUMNS = [
 ]
 
 DISPLAY_METRICS = [
-    "eval_metrics/MASE[0.5]",
+    "eval_metrics/MSE[mean]",
     "eval_metrics/mean_weighted_sum_quantile_loss",
+    "eval_metrics/MASE[0.5]",
     "eval_metrics/MAE[0.5]",
     "eval_metrics/RMSE[mean]",
     "eval_metrics/sMAPE[0.5]",
@@ -39,6 +41,7 @@ DISPLAY_METRICS = [
 ]
 
 METRIC_LABELS = {
+    "eval_metrics/MSE[mean]": "MSE",
     "eval_metrics/MASE[0.5]": "MASE",
     "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS",
     "eval_metrics/MAE[0.5]": "MAE",
@@ -50,8 +53,21 @@ METRIC_LABELS = {
     "eval_metrics/MAPE[0.5]": "MAPE",
 }
 
-VALUE_COLUMNS = [METRIC_LABELS[m] for m in DISPLAY_METRICS]
-RANK_COLUMNS = [f"{METRIC_LABELS[m]}_Rank" for m in DISPLAY_METRICS]
+RANK_SCORE_COLUMN = "RankScore"
+RANK_SCORE_RANK_COLUMN = "RankScore_Rank"
+RANK_SCORE_BASE_METRICS = [
+    "eval_metrics/MSE[mean]",
+    "eval_metrics/mean_weighted_sum_quantile_loss",
+]
+
+VALUE_COLUMNS = ["MSE", "CRPS", RANK_SCORE_COLUMN] + [
+    METRIC_LABELS[m] for m in DISPLAY_METRICS if METRIC_LABELS[m] not in {"MSE", "CRPS"}
+]
+RANK_COLUMNS = ["MSE_Rank", "CRPS_Rank", RANK_SCORE_RANK_COLUMN] + [
+    f"{METRIC_LABELS[m]}_Rank"
+    for m in DISPLAY_METRICS
+    if METRIC_LABELS[m] not in {"MSE", "CRPS"}
+]
 
 LEGACY_MODEL_NAMES = {
     "TSFM1": "Chronos-Bolt-Tiny",
@@ -93,7 +109,10 @@ def format_df(df: pd.DataFrame) -> pd.DataFrame:
     formatted = df.copy()
     for col in formatted.columns:
         if pd.api.types.is_numeric_dtype(formatted[col]):
-            formatted[col] = formatted[col].map(format_number)
+            if col == RANK_SCORE_COLUMN:
+                formatted[col] = formatted[col].map(lambda x: np.nan if pd.isna(x) else f"{x:.1f}")
+            else:
+                formatted[col] = formatted[col].map(format_number)
             formatted[col] = formatted[col].astype(float)
     return formatted
 
@@ -207,6 +226,17 @@ def get_grouped_dfs(
     overall_ranks = overall_ranks.rename(
         columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS}
     )
+    rank_score = compute_rank_score(
+        df,
+        metric_columns=RANK_SCORE_BASE_METRICS,
+        group_columns=("dataset", "term_length", "frequency"),
+    )
+    overall_values = overall_values.join(rank_score, how="left")
+    if RANK_SCORE_COLUMN in overall_values.columns:
+        overall_ranks[RANK_SCORE_RANK_COLUMN] = overall_values[RANK_SCORE_COLUMN].rank(
+            method="average",
+            ascending=False,
+        )
 
     datasets = sorted(df["dataset"].dropna().unique())
     dataset_values: dict[str, pd.DataFrame] = {}
@@ -235,24 +265,28 @@ def prepare_values_table(values_df: pd.DataFrame) -> pd.DataFrame:
     if values_df is None or values_df.empty:
         return pd.DataFrame(columns=["Model"] + VALUE_COLUMNS)
     table = values_df.reset_index().rename(columns={"model": "Model"})
-    table = format_df(table)
     cols = ["Model"] + [col for col in VALUE_COLUMNS if col in table.columns]
     table = table[cols]
-    if "MASE" in table.columns and not table["MASE"].isna().all():
-        return table.sort_values(by=["MASE"])
-    return table
+    if "MSE" in table.columns and not table["MSE"].isna().all():
+        table = table.sort_values(by=["MSE"])
+        return format_df(table)
+    if "RankScore" in table.columns and not table["RankScore"].isna().all():
+        table = table.sort_values(by=["RankScore"], ascending=False)
+    return format_df(table)
 
 
 def prepare_ranks_table(ranks_df: pd.DataFrame) -> pd.DataFrame:
     if ranks_df is None or ranks_df.empty:
         return pd.DataFrame(columns=["Model"] + RANK_COLUMNS)
     table = ranks_df.reset_index().rename(columns={"model": "Model"})
-    table = format_df(table)
     cols = ["Model"] + [col for col in RANK_COLUMNS if col in table.columns]
     table = table[cols]
-    if "MASE_Rank" in table.columns and not table["MASE_Rank"].isna().all():
-        return table.sort_values(by=["MASE_Rank"])
-    return table
+    if "MSE_Rank" in table.columns and not table["MSE_Rank"].isna().all():
+        table = table.sort_values(by=["MSE_Rank"])
+        return format_df(table)
+    if RANK_SCORE_RANK_COLUMN in table.columns and not table[RANK_SCORE_RANK_COLUMN].isna().all():
+        table = table.sort_values(by=[RANK_SCORE_RANK_COLUMN])
+    return format_df(table)
 
 
 DOMAIN_DISPLAY = {
@@ -407,8 +441,9 @@ def dataset_section_title(dataset: str, root_dir: str, *, ranks: bool = False) -
 BASELINE_RANK_HISTORY_COLUMNS = [
     "date",
     "model",
-    "MASE_Rank",
+    "MSE_Rank",
     "CRPS_Rank",
+    "MASE_Rank",
     "MAE_Rank",
     "RMSE_Rank",
     "sMAPE_Rank",
@@ -599,7 +634,7 @@ def make_rank_trend_plot(history_df: pd.DataFrame, metrics: list[str] | None = N
 
     Args:
         history_df: DataFrame from load_baseline_rank_history() with a 'date' column.
-        metrics: List of rank column names to plot. Defaults to MASE_Rank and CRPS_Rank.
+        metrics: List of rank column names to plot. Defaults to MSE_Rank and CRPS_Rank.
 
     Returns None if fewer than 1 data point or plotly unavailable.
     """
@@ -608,7 +643,7 @@ def make_rank_trend_plot(history_df: pd.DataFrame, metrics: list[str] | None = N
         return None
 
     if metrics is None:
-        metrics = ["MASE_Rank", "CRPS_Rank", "MAE_Rank", "RMSE_Rank"]
+        metrics = ["MSE_Rank", "CRPS_Rank", "MASE_Rank", "MAE_Rank"]
 
     date_col = "date" if "date" in history_df.columns else "week"
     present_metrics = [m for m in metrics if m in history_df.columns]