Deploy Space: update src

src/about.py

@@ -4,10 +4,12 @@ INTRODUCTION_TEXT = """
 **TSFM Realworld Bench** evaluates time series foundation models on **live TS-Bench
 real-world data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
 Following [GIFT-Eval](https://huggingface.co/spaces/Salesforce/GIFT-Eval), the leaderboard
-reports **absolute metric values** (not normalized to a baseline model) and **per-dataset
-ranks**. The Overall tab also reports **RankScore**, an Elo-style rank-based aggregate
-computed from per-dataset MSE and CRPS ranks. Use the **Overall** tab for aggregate scores;
-each subsequent tab is one **dataset (domain)** with its own absolute results.
+reports **absolute metric values** and **per-dataset ranks**. The Overall tab also reports
+the cumulative **up-to-now live result** with MSE, RMSE, MAPE, quantile CRPS, RTG,
+temporal Stability, Kendall Improvement, Average Rank, Win Rate, and Elo. The
+**GIFT-style Aggregates** tab provides Seasonal-Naive-normalized MSE, CRPS,
+and mean CRPS rank grouped by actual prediction length, domain, and frequency. Each
+subsequent domain tab retains the original absolute per-dataset results.
 """
 
 LLM_BENCHMARKS_TEXT = """
@@ -38,9 +40,18 @@ Once your Pull Request is opened, our automated sandbox pipeline will load your
 ## Metrics
 
 - **MSE** — Mean Squared Error on the mean forecast (absolute)
-- **CRPS** — Mean Weighted Sum Quantile Loss (probabilistic forecast quality)
+- **RMSE** — Root Mean Squared Error on the mean forecast
+- **MAPE** — Mean Absolute Percentage Error, reported only away from zero
+- **CRPS** — quantile approximation of the Continuous Ranked Probability Score
+- **RTG** — normalized real-time MSE gain over causal Seasonal-Naive (higher is better)
+- **Stability** — standard deviation of release-level MSE (lower is better)
+- **Improvement** — Kendall trend statistic over release-level MSE (more negative is better)
+- **Average Rank / Win Rate / Elo** — paired summaries over shared future releases
 - **RankScore** — Elo-style aggregate from per-dataset MSE and CRPS ranks (higher is better)
 - **MSE_Rank** / **CRPS_Rank** — per-dataset rank (lower is better)
+- **Grouped MSE / CRPS** — geometric mean after per-configuration normalization against
+  Seasonal-Naive (lower is better; 1.0 equals the baseline)
+- **Grouped Rank** — mean per-configuration CRPS rank (lower is better)
 """
 
 CITATION_BUTTON_LABEL = "Copy citation"

src/utils.py

@@ -83,6 +83,23 @@ RESULT_COLUMNS = [
     "num_variates",
 ]
 
+GIFT_AGGREGATE_FILES = {
+    "prediction_length": "results_by_prediction_length.csv",
+    "domain": "results_by_domain.csv",
+    "frequency": "results_by_frequency.csv",
+}
+
+GIFT_AGGREGATE_LABELS = {
+    "prediction_length": "Prediction Length",
+    "domain": "Domain",
+    "frequency": "Frequency",
+}
+
+LIVE_AGGREGATE_FILES = {
+    "overall": "live_overall_up_to_now.csv",
+    "rank": "live_rank_up_to_now.csv",
+}
+
 
 def format_number(value):
     if isinstance(value, (int, float)):
@@ -289,6 +306,183 @@ def prepare_ranks_table(ranks_df: pd.DataFrame) -> pd.DataFrame:
     return format_df(table)
 
 
+def load_gift_aggregate_table(
+    root_dir: str = "results",
+    dimension: str = "prediction_length",
+) -> pd.DataFrame:
+    """Load one persisted GIFT-Eval-style grouped result table."""
+
+    if dimension not in GIFT_AGGREGATE_FILES:
+        raise ValueError(f"Unknown aggregate dimension: {dimension}")
+
+    label = GIFT_AGGREGATE_LABELS[dimension]
+    columns = [label, "Model", "MSE", "CRPS", "Rank", "Configs", "Coverage"]
+    path = Path(root_dir) / "aggregates" / GIFT_AGGREGATE_FILES[dimension]
+    if not path.exists():
+        return pd.DataFrame(columns=columns)
+    try:
+        table = pd.read_csv(path)
+    except (OSError, pd.errors.ParserError, pd.errors.EmptyDataError):
+        return pd.DataFrame(columns=columns)
+
+    required = {
+        dimension,
+        "model",
+        "MSE",
+        "CRPS",
+        "Rank",
+        "n_configs",
+        "n_group_configs",
+        "coverage",
+    }
+    if table.empty or not required.issubset(table.columns):
+        return pd.DataFrame(columns=columns)
+
+    table = table.sort_values([dimension, "Rank", "model"]).copy()
+    table["Configs"] = (
+        table["n_configs"].fillna(0).astype(int).astype(str)
+        + "/"
+        + table["n_group_configs"].fillna(0).astype(int).astype(str)
+    )
+    table["Coverage"] = table["coverage"].map(
+        lambda value: "n/a" if pd.isna(value) else f"{100.0 * float(value):.1f}%"
+    )
+    for metric in ("MSE", "CRPS", "Rank"):
+        table[metric] = pd.to_numeric(table[metric], errors="coerce").round(3)
+    table = table.rename(columns={dimension: label, "model": "Model"})
+    return table.loc[:, columns].reset_index(drop=True)
+
+
+def load_gift_aggregate_metadata_md(root_dir: str = "results") -> str:
+    path = Path(root_dir) / "aggregates" / "metadata.json"
+    if not path.exists():
+        return "Aggregate metadata is not available yet."
+    try:
+        metadata = json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError):
+        return "Aggregate metadata could not be loaded."
+    generated = format_timestamp_utc8(metadata.get("generated_at", ""))
+    return (
+        f"**Protocol:** per-configuration normalization against "
+        f"`{metadata.get('baseline_model', 'Seasonal-Naive')}`; geometric mean for "
+        f"MSE/CRPS; mean CRPS rank. **Coverage:** "
+        f"{metadata.get('num_models', 0)} models × {metadata.get('num_configs', 0)} "
+        f"configurations. **Generated:** {generated}."
+    )
+
+
+def load_live_aggregate_table(
+    root_dir: str = "results",
+    table: str = "overall",
+) -> pd.DataFrame:
+    """Load the cumulative future-release metric or rank table."""
+
+    if table not in LIVE_AGGREGATE_FILES:
+        raise ValueError(f"Unknown live aggregate table: {table}")
+    if table == "overall":
+        columns = [
+            "Rank",
+            "Model",
+            "MSE",
+            "RMSE",
+            "MAPE",
+            "CRPS",
+            "RTG",
+            "Stability",
+            "Improvement",
+            "Datasets",
+            "Releases",
+        ]
+    else:
+        columns = [
+            "Rank",
+            "Model",
+            "Average Rank",
+            "Win Rate",
+            "Elo",
+            "Dataset Pair Matches",
+            "Shared Releases",
+            "Pairwise Matches",
+        ]
+
+    path = Path(root_dir) / "aggregates" / LIVE_AGGREGATE_FILES[table]
+    if not path.exists():
+        return pd.DataFrame(columns=columns)
+    try:
+        frame = pd.read_csv(path)
+    except (OSError, pd.errors.ParserError, pd.errors.EmptyDataError):
+        return pd.DataFrame(columns=columns)
+    if frame.empty:
+        return pd.DataFrame(columns=columns)
+
+    if table == "overall":
+        source_columns = [
+            "Rank",
+            "model",
+            "MSE",
+            "RMSE",
+            "MAPE",
+            "CRPS",
+            "RTG",
+            "Stability",
+            "Improvement",
+            "n_datasets",
+            "n_releases",
+        ]
+        if not set(source_columns).issubset(frame.columns):
+            return pd.DataFrame(columns=columns)
+        frame = frame[source_columns].rename(
+            columns={"model": "Model", "n_datasets": "Datasets", "n_releases": "Releases"}
+        )
+        for metric in ("MSE", "RMSE", "MAPE", "CRPS", "RTG", "Stability", "Improvement"):
+            frame[metric] = pd.to_numeric(frame[metric], errors="coerce").round(4)
+    else:
+        source_columns = [
+            "Rank",
+            "model",
+            "AverageRank",
+            "WinRate",
+            "Elo",
+            "dataset_pair_matches",
+            "shared_releases",
+            "pairwise_matches",
+        ]
+        if not set(source_columns).issubset(frame.columns):
+            return pd.DataFrame(columns=columns)
+        frame = frame[source_columns].rename(
+            columns={
+                "model": "Model",
+                "AverageRank": "Average Rank",
+                "WinRate": "Win Rate",
+                "dataset_pair_matches": "Dataset Pair Matches",
+                "shared_releases": "Shared Releases",
+                "pairwise_matches": "Pairwise Matches",
+            }
+        )
+        for metric in ("Average Rank", "Win Rate", "Elo"):
+            frame[metric] = pd.to_numeric(frame[metric], errors="coerce").round(4)
+    return frame.loc[:, columns].reset_index(drop=True)
+
+
+def load_live_aggregate_metadata_md(root_dir: str = "results") -> str:
+    path = Path(root_dir) / "aggregates" / "live_metadata.json"
+    if not path.exists():
+        return "Cumulative live metrics are not available yet."
+    try:
+        metadata = json.loads(path.read_text())
+    except (OSError, json.JSONDecodeError):
+        return "Cumulative live metric metadata could not be loaded."
+    generated = format_timestamp_utc8(metadata.get("generated_at", ""))
+    return (
+        f"**Up to now:** {metadata.get('num_models', 0)} models, "
+        f"{metadata.get('num_datasets', 0)} datasets, and "
+        f"{metadata.get('num_unique_releases', 0)} distinct future releases. "
+        "MSE/RMSE/CRPS use causal context z-normalization; RTG is relative to "
+        "24-step Seasonal-Naive. **Generated:** "
+        f"{generated}."
+    )
+
+
 DOMAIN_DISPLAY = {
     "Weather": "Climate",
     "Air Quality": "Climate",