Spaces:
Running
Running
zhouziyu02 commited on
Commit ·
6ae0ddf
1
Parent(s): d5f690f
Update online eval results (6 models)
Browse files- app.py +25 -20
- results/chronos_2/online_meta.json +4 -4
- results/chronos_bolt_base/online_meta.json +4 -4
- results/chronos_bolt_mini/all_results.csv +4 -0
- results/chronos_bolt_mini/config.json +11 -0
- results/chronos_bolt_mini/online_meta.json +25 -0
- results/chronos_bolt_tiny/online_meta.json +4 -4
- results/moirai_1_1_small/all_results.csv +4 -0
- results/moirai_1_1_small/config.json +11 -0
- results/moirai_1_1_small/online_meta.json +25 -0
- results/online_status.json +18 -42
- results/timesfm_2_5/all_results.csv +4 -0
- results/timesfm_2_5/config.json +11 -0
- results/timesfm_2_5/online_meta.json +25 -0
- src/about.py +3 -1
- src/utils.py +65 -48
app.py
CHANGED
|
@@ -12,7 +12,12 @@ from src.about import (
|
|
| 12 |
TITLE,
|
| 13 |
)
|
| 14 |
from src.display.css_html_js import custom_css
|
| 15 |
-
from src.utils import
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results")
|
| 18 |
REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
|
|
@@ -25,22 +30,13 @@ def load_online_status(results_path: str = RESULTS_PATH) -> dict:
|
|
| 25 |
return json.loads(status_path.read_text())
|
| 26 |
|
| 27 |
|
| 28 |
-
def build_overall_table(result_df):
|
| 29 |
-
table = result_df.rename(columns={"model": "Model"})
|
| 30 |
-
if "MASE_Rank" in table.columns:
|
| 31 |
-
table = table.sort_values(by=["MASE_Rank"])
|
| 32 |
-
|
| 33 |
-
display_cols = ["Model", "MASE", "MASE_Rank", "CRPS", "CRPS_Rank"]
|
| 34 |
-
cols = [col for col in display_cols if col in table.columns]
|
| 35 |
-
return table[cols].reset_index(drop=True)
|
| 36 |
-
|
| 37 |
-
|
| 38 |
def refresh_leaderboard():
|
| 39 |
grouped = get_grouped_dfs(RESULTS_PATH)
|
| 40 |
status = load_online_status(RESULTS_PATH)
|
| 41 |
baseline = load_baseline_name(RESULTS_PATH)
|
| 42 |
|
| 43 |
-
|
|
|
|
| 44 |
|
| 45 |
finished = status.get("finished_at", "n/a")
|
| 46 |
data_source = status.get("data_source", "n/a")
|
|
@@ -49,10 +45,10 @@ def refresh_leaderboard():
|
|
| 49 |
f"Baseline: `{baseline}` · Last eval: `{finished}` · "
|
| 50 |
f"Auto-refresh every {REFRESH_SECONDS}s"
|
| 51 |
)
|
| 52 |
-
return
|
| 53 |
|
| 54 |
|
| 55 |
-
|
| 56 |
|
| 57 |
demo = gr.Blocks(css=custom_css)
|
| 58 |
with demo:
|
|
@@ -63,11 +59,18 @@ with demo:
|
|
| 63 |
with gr.Row():
|
| 64 |
refresh_btn = gr.Button("Refresh now", variant="secondary")
|
| 65 |
|
| 66 |
-
|
| 67 |
-
value=
|
| 68 |
interactive=False,
|
| 69 |
wrap=True,
|
| 70 |
-
label="
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
)
|
| 72 |
|
| 73 |
with gr.Accordion("About", open=False):
|
|
@@ -81,8 +84,10 @@ with demo:
|
|
| 81 |
show_copy_button=True,
|
| 82 |
)
|
| 83 |
|
| 84 |
-
refresh_btn.click(refresh_leaderboard, outputs=[
|
| 85 |
-
demo.load(refresh_leaderboard, outputs=[
|
| 86 |
-
gr.Timer(value=REFRESH_SECONDS).tick(
|
|
|
|
|
|
|
| 87 |
|
| 88 |
demo.queue(default_concurrency_limit=20).launch()
|
|
|
|
| 12 |
TITLE,
|
| 13 |
)
|
| 14 |
from src.display.css_html_js import custom_css
|
| 15 |
+
from src.utils import (
|
| 16 |
+
get_grouped_dfs,
|
| 17 |
+
load_baseline_name,
|
| 18 |
+
prepare_ranks_table,
|
| 19 |
+
prepare_values_table,
|
| 20 |
+
)
|
| 21 |
|
| 22 |
RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results")
|
| 23 |
REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
|
|
|
|
| 30 |
return json.loads(status_path.read_text())
|
| 31 |
|
| 32 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
def refresh_leaderboard():
|
| 34 |
grouped = get_grouped_dfs(RESULTS_PATH)
|
| 35 |
status = load_online_status(RESULTS_PATH)
|
| 36 |
baseline = load_baseline_name(RESULTS_PATH)
|
| 37 |
|
| 38 |
+
values_table = prepare_values_table(grouped["overall_values"])
|
| 39 |
+
ranks_table = prepare_ranks_table(grouped["overall_ranks"])
|
| 40 |
|
| 41 |
finished = status.get("finished_at", "n/a")
|
| 42 |
data_source = status.get("data_source", "n/a")
|
|
|
|
| 45 |
f"Baseline: `{baseline}` · Last eval: `{finished}` · "
|
| 46 |
f"Auto-refresh every {REFRESH_SECONDS}s"
|
| 47 |
)
|
| 48 |
+
return values_table, ranks_table, status_line
|
| 49 |
|
| 50 |
|
| 51 |
+
values_table, ranks_table, status_md = refresh_leaderboard()
|
| 52 |
|
| 53 |
demo = gr.Blocks(css=custom_css)
|
| 54 |
with demo:
|
|
|
|
| 59 |
with gr.Row():
|
| 60 |
refresh_btn = gr.Button("Refresh now", variant="secondary")
|
| 61 |
|
| 62 |
+
values_df = gr.Dataframe(
|
| 63 |
+
value=values_table,
|
| 64 |
interactive=False,
|
| 65 |
wrap=True,
|
| 66 |
+
label="Metric values (normalized vs baseline, lower is better)",
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
ranks_df = gr.Dataframe(
|
| 70 |
+
value=ranks_table,
|
| 71 |
+
interactive=False,
|
| 72 |
+
wrap=True,
|
| 73 |
+
label="Ranks (average per-dataset rank, lower is better)",
|
| 74 |
)
|
| 75 |
|
| 76 |
with gr.Accordion("About", open=False):
|
|
|
|
| 84 |
show_copy_button=True,
|
| 85 |
)
|
| 86 |
|
| 87 |
+
refresh_btn.click(refresh_leaderboard, outputs=[values_df, ranks_df, status])
|
| 88 |
+
demo.load(refresh_leaderboard, outputs=[values_df, ranks_df, status])
|
| 89 |
+
gr.Timer(value=REFRESH_SECONDS).tick(
|
| 90 |
+
refresh_leaderboard, outputs=[values_df, ranks_df, status]
|
| 91 |
+
)
|
| 92 |
|
| 93 |
demo.queue(default_concurrency_limit=20).launch()
|
results/chronos_2/online_meta.json
CHANGED
|
@@ -1,23 +1,23 @@
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-2",
|
| 3 |
"api_model_id": "amazon/chronos-2",
|
| 4 |
-
"evaluated_at": "2026-05-
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
"dataset": "berlin/H",
|
| 8 |
-
"data_fetched_at": "2026-05-
|
| 9 |
"context_length": 504,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
"dataset": "nyc/H",
|
| 14 |
-
"data_fetched_at": "2026-05-
|
| 15 |
"context_length": 504,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "tokyo/H",
|
| 20 |
-
"data_fetched_at": "2026-05-
|
| 21 |
"context_length": 504,
|
| 22 |
"prediction_length": 24
|
| 23 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-2",
|
| 3 |
"api_model_id": "amazon/chronos-2",
|
| 4 |
+
"evaluated_at": "2026-05-28T10:12:27.369342+00:00",
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
"dataset": "berlin/H",
|
| 8 |
+
"data_fetched_at": "2026-05-28T10:12:07.138621+00:00",
|
| 9 |
"context_length": 504,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
"dataset": "nyc/H",
|
| 14 |
+
"data_fetched_at": "2026-05-28T10:12:16.626231+00:00",
|
| 15 |
"context_length": 504,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "tokyo/H",
|
| 20 |
+
"data_fetched_at": "2026-05-28T10:12:22.950374+00:00",
|
| 21 |
"context_length": 504,
|
| 22 |
"prediction_length": 24
|
| 23 |
}
|
results/chronos_bolt_base/online_meta.json
CHANGED
|
@@ -1,23 +1,23 @@
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Base",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-base",
|
| 4 |
-
"evaluated_at": "2026-05-
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
"dataset": "berlin/H",
|
| 8 |
-
"data_fetched_at": "2026-05-
|
| 9 |
"context_length": 504,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
"dataset": "nyc/H",
|
| 14 |
-
"data_fetched_at": "2026-05-
|
| 15 |
"context_length": 504,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "tokyo/H",
|
| 20 |
-
"data_fetched_at": "2026-05-
|
| 21 |
"context_length": 504,
|
| 22 |
"prediction_length": 24
|
| 23 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Base",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-base",
|
| 4 |
+
"evaluated_at": "2026-05-28T10:12:05.182036+00:00",
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
"dataset": "berlin/H",
|
| 8 |
+
"data_fetched_at": "2026-05-28T10:10:35.736508+00:00",
|
| 9 |
"context_length": 504,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
"dataset": "nyc/H",
|
| 14 |
+
"data_fetched_at": "2026-05-28T10:11:53.331496+00:00",
|
| 15 |
"context_length": 504,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "tokyo/H",
|
| 20 |
+
"data_fetched_at": "2026-05-28T10:12:00.889157+00:00",
|
| 21 |
"context_length": 504,
|
| 22 |
"prediction_length": 24
|
| 23 |
}
|
results/chronos_bolt_mini/all_results.csv
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
berlin/H/short,Chronos-Bolt-Mini,23.02803795462124,23.02803795462124,3.1608052717314825,0.6035438633560637,0.15491954678594283,0.1448325903393575,3.115631327236734,4.798753791831921,0.19212093626207102,0.1265447019142281,0.1081969714859019,Energy,1
|
| 3 |
+
nyc/H/short,Chronos-Bolt-Mini,67.67541915984299,67.67541915984299,5.3830602169036865,0.6020463455851162,0.13388074766003255,0.11707076108417061,4.7290140050230995,8.226507105682398,0.022892831572236993,0.014980050379299257,0.01192178705488713,Econ/Fin,1
|
| 4 |
+
tokyo/H/short,Chronos-Bolt-Mini,5.129979933399418,5.129979933399418,1.3517586653534737,0.5165812565468629,0.37651829322351843,0.8238071342871242,3.8948858261463672,2.264945900766598,0.250040044563296,0.14922819869975965,0.106032322595721,Transport,1
|
results/chronos_bolt_mini/config.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "Chronos-Bolt-Mini",
|
| 3 |
+
"model_type": "zero-shot",
|
| 4 |
+
"model_dtype": "float32",
|
| 5 |
+
"model_link": "https://tsfm.ai/models/amazon/chronos-bolt-mini",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
|
| 7 |
+
"org": "Amazon / TSFM.ai",
|
| 8 |
+
"testdata_leakage": "No",
|
| 9 |
+
"replication_code_available": "Yes",
|
| 10 |
+
"api_model_id": "amazon/chronos-bolt-mini"
|
| 11 |
+
}
|
results/chronos_bolt_mini/online_meta.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "Chronos-Bolt-Mini",
|
| 3 |
+
"api_model_id": "amazon/chronos-bolt-mini",
|
| 4 |
+
"evaluated_at": "2026-05-28T10:10:33.484071+00:00",
|
| 5 |
+
"datasets": [
|
| 6 |
+
{
|
| 7 |
+
"dataset": "berlin/H",
|
| 8 |
+
"data_fetched_at": "2026-05-28T10:10:08.921860+00:00",
|
| 9 |
+
"context_length": 504,
|
| 10 |
+
"prediction_length": 24
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"dataset": "nyc/H",
|
| 14 |
+
"data_fetched_at": "2026-05-28T10:10:20.642074+00:00",
|
| 15 |
+
"context_length": 504,
|
| 16 |
+
"prediction_length": 24
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"dataset": "tokyo/H",
|
| 20 |
+
"data_fetched_at": "2026-05-28T10:10:28.780465+00:00",
|
| 21 |
+
"context_length": 504,
|
| 22 |
+
"prediction_length": 24
|
| 23 |
+
}
|
| 24 |
+
]
|
| 25 |
+
}
|
results/chronos_bolt_tiny/online_meta.json
CHANGED
|
@@ -1,23 +1,23 @@
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Tiny",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-tiny",
|
| 4 |
-
"evaluated_at": "2026-05-
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
"dataset": "berlin/H",
|
| 8 |
-
"data_fetched_at": "2026-05-
|
| 9 |
"context_length": 504,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
"dataset": "nyc/H",
|
| 14 |
-
"data_fetched_at": "2026-05-
|
| 15 |
"context_length": 504,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "tokyo/H",
|
| 20 |
-
"data_fetched_at": "2026-05-
|
| 21 |
"context_length": 504,
|
| 22 |
"prediction_length": 24
|
| 23 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"model": "Chronos-Bolt-Tiny",
|
| 3 |
"api_model_id": "amazon/chronos-bolt-tiny",
|
| 4 |
+
"evaluated_at": "2026-05-28T10:09:59.581604+00:00",
|
| 5 |
"datasets": [
|
| 6 |
{
|
| 7 |
"dataset": "berlin/H",
|
| 8 |
+
"data_fetched_at": "2026-05-28T10:09:00.615970+00:00",
|
| 9 |
"context_length": 504,
|
| 10 |
"prediction_length": 24
|
| 11 |
},
|
| 12 |
{
|
| 13 |
"dataset": "nyc/H",
|
| 14 |
+
"data_fetched_at": "2026-05-28T10:09:46.973725+00:00",
|
| 15 |
"context_length": 504,
|
| 16 |
"prediction_length": 24
|
| 17 |
},
|
| 18 |
{
|
| 19 |
"dataset": "tokyo/H",
|
| 20 |
+
"data_fetched_at": "2026-05-28T10:09:54.263751+00:00",
|
| 21 |
"context_length": 504,
|
| 22 |
"prediction_length": 24
|
| 23 |
}
|
results/moirai_1_1_small/all_results.csv
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
berlin/H/short,Moirai-1.1-Small,25.942115142478016,25.914808326787288,4.134999871253967,0.8981191360243419,0.2807477289705378,0.2263154350433037,4.74372356414242,5.0933402735805915,0.20391487967713806,0.16554715685998675,0.13114703596375427,Energy,1
|
| 3 |
+
nyc/H/short,Moirai-1.1-Small,115.85427334839227,120.1817511764708,7.214917302131653,0.797558713399391,0.19310537226504765,0.15686588488083114,4.257123747918823,10.763562298253877,0.029952981945521127,0.02007776623583399,0.014418479379084874,Econ/Fin,1
|
| 4 |
+
tokyo/H/short,Moirai-1.1-Small,5.042828195691895,4.650267012598303,1.3240045196531962,0.49590462426539694,0.38585838156843166,0.8294245926005037,4.263774493409405,2.2456242329677276,0.2479070175996799,0.1461642633424671,0.11333818582137908,Transport,1
|
results/moirai_1_1_small/config.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "Moirai-1.1-Small",
|
| 3 |
+
"model_type": "zero-shot",
|
| 4 |
+
"model_dtype": "float32",
|
| 5 |
+
"model_link": "https://tsfm.ai/models/Salesforce/moirai-1.1-R-small",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
|
| 7 |
+
"org": "Salesforce / TSFM.ai",
|
| 8 |
+
"testdata_leakage": "No",
|
| 9 |
+
"replication_code_available": "Yes",
|
| 10 |
+
"api_model_id": "Salesforce/moirai-1.1-R-small"
|
| 11 |
+
}
|
results/moirai_1_1_small/online_meta.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "Moirai-1.1-Small",
|
| 3 |
+
"api_model_id": "Salesforce/moirai-1.1-R-small",
|
| 4 |
+
"evaluated_at": "2026-05-28T10:18:27.617821+00:00",
|
| 5 |
+
"datasets": [
|
| 6 |
+
{
|
| 7 |
+
"dataset": "berlin/H",
|
| 8 |
+
"data_fetched_at": "2026-05-28T10:18:02.944773+00:00",
|
| 9 |
+
"context_length": 504,
|
| 10 |
+
"prediction_length": 24
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"dataset": "nyc/H",
|
| 14 |
+
"data_fetched_at": "2026-05-28T10:18:14.313951+00:00",
|
| 15 |
+
"context_length": 504,
|
| 16 |
+
"prediction_length": 24
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"dataset": "tokyo/H",
|
| 20 |
+
"data_fetched_at": "2026-05-28T10:18:22.281838+00:00",
|
| 21 |
+
"context_length": 504,
|
| 22 |
+
"prediction_length": 24
|
| 23 |
+
}
|
| 24 |
+
]
|
| 25 |
+
}
|
results/online_status.json
CHANGED
|
@@ -1,85 +1,61 @@
|
|
| 1 |
{
|
| 2 |
"status": "ok",
|
| 3 |
-
"started_at": "2026-05-
|
| 4 |
-
"finished_at": "2026-05-
|
| 5 |
"data_source": "open_meteo",
|
| 6 |
"data_config": "configs/datasets/open_meteo.yaml",
|
| 7 |
"baseline_model": "Chronos-Bolt-Base",
|
| 8 |
"models": {
|
| 9 |
-
"
|
| 10 |
-
"model": "
|
| 11 |
-
"api_model_id": "
|
| 12 |
-
"evaluated_at": "2026-05-
|
| 13 |
"datasets": [
|
| 14 |
{
|
| 15 |
"dataset": "berlin/H",
|
| 16 |
-
"data_fetched_at": "2026-05-
|
| 17 |
"context_length": 504,
|
| 18 |
"prediction_length": 24
|
| 19 |
},
|
| 20 |
{
|
| 21 |
"dataset": "nyc/H",
|
| 22 |
-
"data_fetched_at": "2026-05-
|
| 23 |
"context_length": 504,
|
| 24 |
"prediction_length": 24
|
| 25 |
},
|
| 26 |
{
|
| 27 |
"dataset": "tokyo/H",
|
| 28 |
-
"data_fetched_at": "2026-05-
|
| 29 |
"context_length": 504,
|
| 30 |
"prediction_length": 24
|
| 31 |
}
|
| 32 |
]
|
| 33 |
},
|
| 34 |
-
"
|
| 35 |
-
"model": "
|
| 36 |
-
"api_model_id": "
|
| 37 |
-
"evaluated_at": "2026-05-
|
| 38 |
"datasets": [
|
| 39 |
{
|
| 40 |
"dataset": "berlin/H",
|
| 41 |
-
"data_fetched_at": "2026-05-
|
| 42 |
"context_length": 504,
|
| 43 |
"prediction_length": 24
|
| 44 |
},
|
| 45 |
{
|
| 46 |
"dataset": "nyc/H",
|
| 47 |
-
"data_fetched_at": "2026-05-
|
| 48 |
"context_length": 504,
|
| 49 |
"prediction_length": 24
|
| 50 |
},
|
| 51 |
{
|
| 52 |
"dataset": "tokyo/H",
|
| 53 |
-
"data_fetched_at": "2026-05-
|
| 54 |
-
"context_length": 504,
|
| 55 |
-
"prediction_length": 24
|
| 56 |
-
}
|
| 57 |
-
]
|
| 58 |
-
},
|
| 59 |
-
"Chronos-2": {
|
| 60 |
-
"model": "Chronos-2",
|
| 61 |
-
"api_model_id": "amazon/chronos-2",
|
| 62 |
-
"evaluated_at": "2026-05-28T09:58:41.598566+00:00",
|
| 63 |
-
"datasets": [
|
| 64 |
-
{
|
| 65 |
-
"dataset": "berlin/H",
|
| 66 |
-
"data_fetched_at": "2026-05-28T09:58:16.749981+00:00",
|
| 67 |
-
"context_length": 504,
|
| 68 |
-
"prediction_length": 24
|
| 69 |
-
},
|
| 70 |
-
{
|
| 71 |
-
"dataset": "nyc/H",
|
| 72 |
-
"data_fetched_at": "2026-05-28T09:58:28.553123+00:00",
|
| 73 |
-
"context_length": 504,
|
| 74 |
-
"prediction_length": 24
|
| 75 |
-
},
|
| 76 |
-
{
|
| 77 |
-
"dataset": "tokyo/H",
|
| 78 |
-
"data_fetched_at": "2026-05-28T09:58:37.331184+00:00",
|
| 79 |
"context_length": 504,
|
| 80 |
"prediction_length": 24
|
| 81 |
}
|
| 82 |
]
|
| 83 |
}
|
| 84 |
-
}
|
|
|
|
| 85 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"status": "ok",
|
| 3 |
+
"started_at": "2026-05-28T10:17:24.399004+00:00",
|
| 4 |
+
"finished_at": "2026-05-28T10:18:27.618686+00:00",
|
| 5 |
"data_source": "open_meteo",
|
| 6 |
"data_config": "configs/datasets/open_meteo.yaml",
|
| 7 |
"baseline_model": "Chronos-Bolt-Base",
|
| 8 |
"models": {
|
| 9 |
+
"TimesFM-2.5": {
|
| 10 |
+
"model": "TimesFM-2.5",
|
| 11 |
+
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 12 |
+
"evaluated_at": "2026-05-28T10:18:01.120947+00:00",
|
| 13 |
"datasets": [
|
| 14 |
{
|
| 15 |
"dataset": "berlin/H",
|
| 16 |
+
"data_fetched_at": "2026-05-28T10:17:26.673802+00:00",
|
| 17 |
"context_length": 504,
|
| 18 |
"prediction_length": 24
|
| 19 |
},
|
| 20 |
{
|
| 21 |
"dataset": "nyc/H",
|
| 22 |
+
"data_fetched_at": "2026-05-28T10:17:42.004619+00:00",
|
| 23 |
"context_length": 504,
|
| 24 |
"prediction_length": 24
|
| 25 |
},
|
| 26 |
{
|
| 27 |
"dataset": "tokyo/H",
|
| 28 |
+
"data_fetched_at": "2026-05-28T10:17:52.661277+00:00",
|
| 29 |
"context_length": 504,
|
| 30 |
"prediction_length": 24
|
| 31 |
}
|
| 32 |
]
|
| 33 |
},
|
| 34 |
+
"Moirai-1.1-Small": {
|
| 35 |
+
"model": "Moirai-1.1-Small",
|
| 36 |
+
"api_model_id": "Salesforce/moirai-1.1-R-small",
|
| 37 |
+
"evaluated_at": "2026-05-28T10:18:27.617821+00:00",
|
| 38 |
"datasets": [
|
| 39 |
{
|
| 40 |
"dataset": "berlin/H",
|
| 41 |
+
"data_fetched_at": "2026-05-28T10:18:02.944773+00:00",
|
| 42 |
"context_length": 504,
|
| 43 |
"prediction_length": 24
|
| 44 |
},
|
| 45 |
{
|
| 46 |
"dataset": "nyc/H",
|
| 47 |
+
"data_fetched_at": "2026-05-28T10:18:14.313951+00:00",
|
| 48 |
"context_length": 504,
|
| 49 |
"prediction_length": 24
|
| 50 |
},
|
| 51 |
{
|
| 52 |
"dataset": "tokyo/H",
|
| 53 |
+
"data_fetched_at": "2026-05-28T10:18:22.281838+00:00",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
"context_length": 504,
|
| 55 |
"prediction_length": 24
|
| 56 |
}
|
| 57 |
]
|
| 58 |
}
|
| 59 |
+
},
|
| 60 |
+
"failed_models": []
|
| 61 |
}
|
results/timesfm_2_5/all_results.csv
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
|
| 2 |
+
berlin/H/short,TimesFM-2.5,26.54830902205753,31.322212017679387,3.443761388460795,0.6303993503146956,0.1386495615012084,0.13621565370038174,18.01169611505954,5.152505121012257,0.20628357921361393,0.13787301681124278,0.1312424613907817,Energy,1
|
| 3 |
+
nyc/H/short,TimesFM-2.5,66.51865488618301,44.39263497040171,4.487805022133721,0.4947887756747412,0.09551055813760116,0.08531003986906531,15.677110160428015,8.155896939404213,0.02269633667797287,0.012488722513809137,0.014150956333775139,Econ/Fin,1
|
| 4 |
+
tokyo/H/short,TimesFM-2.5,4.1984207631854,2.872721300592553,1.081747318436909,0.411514142106336,0.31230229584942987,0.6226950610386188,18.191892843171864,2.0490048226359545,0.2262011013105863,0.11942013609095183,0.13494832121177208,Transport,1
|
results/timesfm_2_5/config.json
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "TimesFM-2.5",
|
| 3 |
+
"model_type": "zero-shot",
|
| 4 |
+
"model_dtype": "float32",
|
| 5 |
+
"model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
|
| 6 |
+
"code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
|
| 7 |
+
"org": "Google / TSFM.ai",
|
| 8 |
+
"testdata_leakage": "No",
|
| 9 |
+
"replication_code_available": "Yes",
|
| 10 |
+
"api_model_id": "google/timesfm-2.5-200m-pytorch"
|
| 11 |
+
}
|
results/timesfm_2_5/online_meta.json
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "TimesFM-2.5",
|
| 3 |
+
"api_model_id": "google/timesfm-2.5-200m-pytorch",
|
| 4 |
+
"evaluated_at": "2026-05-28T10:18:01.120947+00:00",
|
| 5 |
+
"datasets": [
|
| 6 |
+
{
|
| 7 |
+
"dataset": "berlin/H",
|
| 8 |
+
"data_fetched_at": "2026-05-28T10:17:26.673802+00:00",
|
| 9 |
+
"context_length": 504,
|
| 10 |
+
"prediction_length": 24
|
| 11 |
+
},
|
| 12 |
+
{
|
| 13 |
+
"dataset": "nyc/H",
|
| 14 |
+
"data_fetched_at": "2026-05-28T10:17:42.004619+00:00",
|
| 15 |
+
"context_length": 504,
|
| 16 |
+
"prediction_length": 24
|
| 17 |
+
},
|
| 18 |
+
{
|
| 19 |
+
"dataset": "tokyo/H",
|
| 20 |
+
"data_fetched_at": "2026-05-28T10:17:52.661277+00:00",
|
| 21 |
+
"context_length": 504,
|
| 22 |
+
"prediction_length": 24
|
| 23 |
+
}
|
| 24 |
+
]
|
| 25 |
+
}
|
src/about.py
CHANGED
|
@@ -3,7 +3,9 @@ TITLE = """ TSFM Realworld Bench Leaderboard """
|
|
| 3 |
INTRODUCTION_TEXT = """
|
| 4 |
**TSFM Realworld Bench** evaluates time series foundation models on **live Open-Meteo
|
| 5 |
weather data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
|
| 6 |
-
Scores are normalized against **Chronos-Bolt-Base**
|
|
|
|
|
|
|
| 7 |
"""
|
| 8 |
|
| 9 |
LLM_BENCHMARKS_TEXT = """
|
|
|
|
| 3 |
INTRODUCTION_TEXT = """
|
| 4 |
**TSFM Realworld Bench** evaluates time series foundation models on **live Open-Meteo
|
| 5 |
weather data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
|
| 6 |
+
Scores are normalized against **Chronos-Bolt-Base**. The leaderboard shows
|
| 7 |
+
**metric values** and **ranks** in separate tables (MASE, CRPS, MAE, RMSE, sMAPE,
|
| 8 |
+
MSIS, ND, NRMSE, MAPE).
|
| 9 |
"""
|
| 10 |
|
| 11 |
LLM_BENCHMARKS_TEXT = """
|
src/utils.py
CHANGED
|
@@ -6,8 +6,6 @@ from pathlib import Path
|
|
| 6 |
import pandas as pd
|
| 7 |
from scipy import stats
|
| 8 |
|
| 9 |
-
from src.leaderboard.read_evals import get_model_info
|
| 10 |
-
|
| 11 |
METRIC_COLUMNS = [
|
| 12 |
"eval_metrics/MSE[mean]",
|
| 13 |
"eval_metrics/MSE[0.5]",
|
|
@@ -21,9 +19,33 @@ METRIC_COLUMNS = [
|
|
| 21 |
"eval_metrics/ND[0.5]",
|
| 22 |
"eval_metrics/mean_weighted_sum_quantile_loss",
|
| 23 |
]
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
LEGACY_MODEL_NAMES = {
|
| 29 |
"TSFM1": "Chronos-Bolt-Tiny",
|
|
@@ -49,17 +71,6 @@ def format_df(df: pd.DataFrame) -> pd.DataFrame:
|
|
| 49 |
return formatted
|
| 50 |
|
| 51 |
|
| 52 |
-
def rename_metrics(df: pd.DataFrame) -> pd.DataFrame:
|
| 53 |
-
return df.rename(
|
| 54 |
-
columns={
|
| 55 |
-
"eval_metrics/MASE[0.5]": "MASE",
|
| 56 |
-
"eval_metrics/mean_weighted_sum_quantile_loss": "CRPS",
|
| 57 |
-
"rank": "CRPS_Rank",
|
| 58 |
-
"Rank_MASE": "MASE_Rank",
|
| 59 |
-
}
|
| 60 |
-
)
|
| 61 |
-
|
| 62 |
-
|
| 63 |
def unify_freq(df: pd.DataFrame) -> pd.DataFrame:
|
| 64 |
df = df.copy()
|
| 65 |
df["frequency"] = df["frequency"].str.replace(r"\d+", "", regex=True)
|
|
@@ -88,7 +99,8 @@ def standardize_df(df: pd.DataFrame, root_dir: str = "results") -> pd.DataFrame:
|
|
| 88 |
if baseline_name not in set(df["model"]):
|
| 89 |
return df
|
| 90 |
|
| 91 |
-
|
|
|
|
| 92 |
baseline = group[group["model"] == baseline_name]
|
| 93 |
if baseline.empty:
|
| 94 |
continue
|
|
@@ -131,46 +143,51 @@ def enrich_with_properties(df: pd.DataFrame, ds_properties_path: str) -> pd.Data
|
|
| 131 |
return unify_freq(df)
|
| 132 |
|
| 133 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
def get_grouped_dfs(
|
| 135 |
root_dir: str = "results",
|
| 136 |
ds_properties: str | None = None,
|
| 137 |
) -> dict[str, pd.DataFrame]:
|
| 138 |
if ds_properties is None:
|
| 139 |
ds_properties = str(Path(root_dir) / "dataset_properties.csv")
|
|
|
|
| 140 |
df = load_all_results(root_dir)
|
| 141 |
df = enrich_with_properties(df, ds_properties)
|
| 142 |
df = standardize_df(df, root_dir)
|
|
|
|
| 143 |
|
| 144 |
-
|
| 145 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
)
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
return {"overall": overall}
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
def get_merged_df(result_df: pd.DataFrame, model_info_df: pd.DataFrame) -> pd.DataFrame:
|
| 167 |
-
merged = pd.merge(model_info_df, result_df, left_on="model", right_on="model", how="inner")
|
| 168 |
-
merged = merged.drop(columns=["model"])
|
| 169 |
-
merged = merged.rename(columns={"model_w_link": "Model"})
|
| 170 |
-
return merged
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
def prepare_leaderboard_table(grouped_df: pd.DataFrame) -> pd.DataFrame:
|
| 174 |
-
table = rename_metrics(grouped_df.reset_index())
|
| 175 |
table = format_df(table)
|
| 176 |
-
|
|
|
|
|
|
| 6 |
import pandas as pd
|
| 7 |
from scipy import stats
|
| 8 |
|
|
|
|
|
|
|
| 9 |
METRIC_COLUMNS = [
|
| 10 |
"eval_metrics/MSE[mean]",
|
| 11 |
"eval_metrics/MSE[0.5]",
|
|
|
|
| 19 |
"eval_metrics/ND[0.5]",
|
| 20 |
"eval_metrics/mean_weighted_sum_quantile_loss",
|
| 21 |
]
|
| 22 |
+
|
| 23 |
+
DISPLAY_METRICS = [
|
| 24 |
+
"eval_metrics/MASE[0.5]",
|
| 25 |
+
"eval_metrics/mean_weighted_sum_quantile_loss",
|
| 26 |
+
"eval_metrics/MAE[0.5]",
|
| 27 |
+
"eval_metrics/RMSE[mean]",
|
| 28 |
+
"eval_metrics/sMAPE[0.5]",
|
| 29 |
+
"eval_metrics/MSIS",
|
| 30 |
+
"eval_metrics/ND[0.5]",
|
| 31 |
+
"eval_metrics/NRMSE[mean]",
|
| 32 |
+
"eval_metrics/MAPE[0.5]",
|
| 33 |
+
]
|
| 34 |
+
|
| 35 |
+
METRIC_LABELS = {
|
| 36 |
+
"eval_metrics/MASE[0.5]": "MASE",
|
| 37 |
+
"eval_metrics/mean_weighted_sum_quantile_loss": "CRPS",
|
| 38 |
+
"eval_metrics/MAE[0.5]": "MAE",
|
| 39 |
+
"eval_metrics/RMSE[mean]": "RMSE",
|
| 40 |
+
"eval_metrics/sMAPE[0.5]": "sMAPE",
|
| 41 |
+
"eval_metrics/MSIS": "MSIS",
|
| 42 |
+
"eval_metrics/ND[0.5]": "ND",
|
| 43 |
+
"eval_metrics/NRMSE[mean]": "NRMSE",
|
| 44 |
+
"eval_metrics/MAPE[0.5]": "MAPE",
|
| 45 |
+
}
|
| 46 |
+
|
| 47 |
+
VALUE_COLUMNS = [METRIC_LABELS[m] for m in DISPLAY_METRICS]
|
| 48 |
+
RANK_COLUMNS = [f"{METRIC_LABELS[m]}_Rank" for m in DISPLAY_METRICS]
|
| 49 |
|
| 50 |
LEGACY_MODEL_NAMES = {
|
| 51 |
"TSFM1": "Chronos-Bolt-Tiny",
|
|
|
|
| 71 |
return formatted
|
| 72 |
|
| 73 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
def unify_freq(df: pd.DataFrame) -> pd.DataFrame:
|
| 75 |
df = df.copy()
|
| 76 |
df["frequency"] = df["frequency"].str.replace(r"\d+", "", regex=True)
|
|
|
|
| 99 |
if baseline_name not in set(df["model"]):
|
| 100 |
return df
|
| 101 |
|
| 102 |
+
group_keys = ["dataset", "term_length", "frequency"]
|
| 103 |
+
for _, group in df.groupby(group_keys):
|
| 104 |
baseline = group[group["model"] == baseline_name]
|
| 105 |
if baseline.empty:
|
| 106 |
continue
|
|
|
|
| 143 |
return unify_freq(df)
|
| 144 |
|
| 145 |
|
| 146 |
+
def _add_per_dataset_ranks(df: pd.DataFrame) -> pd.DataFrame:
|
| 147 |
+
df = df.copy()
|
| 148 |
+
group_keys = ["dataset", "term_length", "frequency"]
|
| 149 |
+
for metric in DISPLAY_METRICS:
|
| 150 |
+
rank_col = f"Rank_{metric}"
|
| 151 |
+
df[rank_col] = df.groupby(group_keys)[metric].rank(method="first", ascending=True)
|
| 152 |
+
return df
|
| 153 |
+
|
| 154 |
+
|
| 155 |
def get_grouped_dfs(
|
| 156 |
root_dir: str = "results",
|
| 157 |
ds_properties: str | None = None,
|
| 158 |
) -> dict[str, pd.DataFrame]:
|
| 159 |
if ds_properties is None:
|
| 160 |
ds_properties = str(Path(root_dir) / "dataset_properties.csv")
|
| 161 |
+
|
| 162 |
df = load_all_results(root_dir)
|
| 163 |
df = enrich_with_properties(df, ds_properties)
|
| 164 |
df = standardize_df(df, root_dir)
|
| 165 |
+
df = _add_per_dataset_ranks(df)
|
| 166 |
|
| 167 |
+
rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS]
|
| 168 |
+
overall_values = df.groupby(["model"])[DISPLAY_METRICS].agg(stats.gmean)
|
| 169 |
+
overall_ranks = df.groupby(["model"])[rank_cols].mean()
|
| 170 |
+
|
| 171 |
+
overall_values = overall_values.rename(columns=METRIC_LABELS)
|
| 172 |
+
overall_ranks = overall_ranks.rename(
|
| 173 |
+
columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS}
|
| 174 |
)
|
| 175 |
+
|
| 176 |
+
return {
|
| 177 |
+
"overall_values": overall_values,
|
| 178 |
+
"overall_ranks": overall_ranks,
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def prepare_values_table(values_df: pd.DataFrame) -> pd.DataFrame:
|
| 183 |
+
table = values_df.reset_index().rename(columns={"model": "Model"})
|
| 184 |
+
table = format_df(table)
|
| 185 |
+
cols = ["Model"] + [col for col in VALUE_COLUMNS if col in table.columns]
|
| 186 |
+
return table[cols].sort_values(by=["MASE"])
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def prepare_ranks_table(ranks_df: pd.DataFrame) -> pd.DataFrame:
|
| 190 |
+
table = ranks_df.reset_index().rename(columns={"model": "Model"})
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
table = format_df(table)
|
| 192 |
+
cols = ["Model"] + [col for col in RANK_COLUMNS if col in table.columns]
|
| 193 |
+
return table[cols].sort_values(by=["MASE_Rank"])
|