zhouziyu02 commited on
Commit
6ae0ddf
·
1 Parent(s): d5f690f

Update online eval results (6 models)

Browse files
app.py CHANGED
@@ -12,7 +12,12 @@ from src.about import (
12
  TITLE,
13
  )
14
  from src.display.css_html_js import custom_css
15
- from src.utils import get_grouped_dfs, load_baseline_name, prepare_leaderboard_table
 
 
 
 
 
16
 
17
  RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results")
18
  REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
@@ -25,22 +30,13 @@ def load_online_status(results_path: str = RESULTS_PATH) -> dict:
25
  return json.loads(status_path.read_text())
26
 
27
 
28
- def build_overall_table(result_df):
29
- table = result_df.rename(columns={"model": "Model"})
30
- if "MASE_Rank" in table.columns:
31
- table = table.sort_values(by=["MASE_Rank"])
32
-
33
- display_cols = ["Model", "MASE", "MASE_Rank", "CRPS", "CRPS_Rank"]
34
- cols = [col for col in display_cols if col in table.columns]
35
- return table[cols].reset_index(drop=True)
36
-
37
-
38
  def refresh_leaderboard():
39
  grouped = get_grouped_dfs(RESULTS_PATH)
40
  status = load_online_status(RESULTS_PATH)
41
  baseline = load_baseline_name(RESULTS_PATH)
42
 
43
- overall = build_overall_table(prepare_leaderboard_table(grouped["overall"]))
 
44
 
45
  finished = status.get("finished_at", "n/a")
46
  data_source = status.get("data_source", "n/a")
@@ -49,10 +45,10 @@ def refresh_leaderboard():
49
  f"Baseline: `{baseline}` · Last eval: `{finished}` · "
50
  f"Auto-refresh every {REFRESH_SECONDS}s"
51
  )
52
- return overall, status_line
53
 
54
 
55
- overall_table, status_md = refresh_leaderboard()
56
 
57
  demo = gr.Blocks(css=custom_css)
58
  with demo:
@@ -63,11 +59,18 @@ with demo:
63
  with gr.Row():
64
  refresh_btn = gr.Button("Refresh now", variant="secondary")
65
 
66
- overall_df = gr.Dataframe(
67
- value=overall_table,
68
  interactive=False,
69
  wrap=True,
70
- label="Overall Leaderboard (normalized vs baseline, lower is better)",
 
 
 
 
 
 
 
71
  )
72
 
73
  with gr.Accordion("About", open=False):
@@ -81,8 +84,10 @@ with demo:
81
  show_copy_button=True,
82
  )
83
 
84
- refresh_btn.click(refresh_leaderboard, outputs=[overall_df, status])
85
- demo.load(refresh_leaderboard, outputs=[overall_df, status])
86
- gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, outputs=[overall_df, status])
 
 
87
 
88
  demo.queue(default_concurrency_limit=20).launch()
 
12
  TITLE,
13
  )
14
  from src.display.css_html_js import custom_css
15
+ from src.utils import (
16
+ get_grouped_dfs,
17
+ load_baseline_name,
18
+ prepare_ranks_table,
19
+ prepare_values_table,
20
+ )
21
 
22
  RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "results")
23
  REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300"))
 
30
  return json.loads(status_path.read_text())
31
 
32
 
 
 
 
 
 
 
 
 
 
 
33
  def refresh_leaderboard():
34
  grouped = get_grouped_dfs(RESULTS_PATH)
35
  status = load_online_status(RESULTS_PATH)
36
  baseline = load_baseline_name(RESULTS_PATH)
37
 
38
+ values_table = prepare_values_table(grouped["overall_values"])
39
+ ranks_table = prepare_ranks_table(grouped["overall_ranks"])
40
 
41
  finished = status.get("finished_at", "n/a")
42
  data_source = status.get("data_source", "n/a")
 
45
  f"Baseline: `{baseline}` · Last eval: `{finished}` · "
46
  f"Auto-refresh every {REFRESH_SECONDS}s"
47
  )
48
+ return values_table, ranks_table, status_line
49
 
50
 
51
+ values_table, ranks_table, status_md = refresh_leaderboard()
52
 
53
  demo = gr.Blocks(css=custom_css)
54
  with demo:
 
59
  with gr.Row():
60
  refresh_btn = gr.Button("Refresh now", variant="secondary")
61
 
62
+ values_df = gr.Dataframe(
63
+ value=values_table,
64
  interactive=False,
65
  wrap=True,
66
+ label="Metric values (normalized vs baseline, lower is better)",
67
+ )
68
+
69
+ ranks_df = gr.Dataframe(
70
+ value=ranks_table,
71
+ interactive=False,
72
+ wrap=True,
73
+ label="Ranks (average per-dataset rank, lower is better)",
74
  )
75
 
76
  with gr.Accordion("About", open=False):
 
84
  show_copy_button=True,
85
  )
86
 
87
+ refresh_btn.click(refresh_leaderboard, outputs=[values_df, ranks_df, status])
88
+ demo.load(refresh_leaderboard, outputs=[values_df, ranks_df, status])
89
+ gr.Timer(value=REFRESH_SECONDS).tick(
90
+ refresh_leaderboard, outputs=[values_df, ranks_df, status]
91
+ )
92
 
93
  demo.queue(default_concurrency_limit=20).launch()
results/chronos_2/online_meta.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
  "model": "Chronos-2",
3
  "api_model_id": "amazon/chronos-2",
4
- "evaluated_at": "2026-05-28T09:58:41.598566+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "berlin/H",
8
- "data_fetched_at": "2026-05-28T09:58:16.749981+00:00",
9
  "context_length": 504,
10
  "prediction_length": 24
11
  },
12
  {
13
  "dataset": "nyc/H",
14
- "data_fetched_at": "2026-05-28T09:58:28.553123+00:00",
15
  "context_length": 504,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "tokyo/H",
20
- "data_fetched_at": "2026-05-28T09:58:37.331184+00:00",
21
  "context_length": 504,
22
  "prediction_length": 24
23
  }
 
1
  {
2
  "model": "Chronos-2",
3
  "api_model_id": "amazon/chronos-2",
4
+ "evaluated_at": "2026-05-28T10:12:27.369342+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "berlin/H",
8
+ "data_fetched_at": "2026-05-28T10:12:07.138621+00:00",
9
  "context_length": 504,
10
  "prediction_length": 24
11
  },
12
  {
13
  "dataset": "nyc/H",
14
+ "data_fetched_at": "2026-05-28T10:12:16.626231+00:00",
15
  "context_length": 504,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "tokyo/H",
20
+ "data_fetched_at": "2026-05-28T10:12:22.950374+00:00",
21
  "context_length": 504,
22
  "prediction_length": 24
23
  }
results/chronos_bolt_base/online_meta.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
  "model": "Chronos-Bolt-Base",
3
  "api_model_id": "amazon/chronos-bolt-base",
4
- "evaluated_at": "2026-05-28T09:58:12.886980+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "berlin/H",
8
- "data_fetched_at": "2026-05-28T09:57:42.662907+00:00",
9
  "context_length": 504,
10
  "prediction_length": 24
11
  },
12
  {
13
  "dataset": "nyc/H",
14
- "data_fetched_at": "2026-05-28T09:57:56.819035+00:00",
15
  "context_length": 504,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "tokyo/H",
20
- "data_fetched_at": "2026-05-28T09:58:06.485785+00:00",
21
  "context_length": 504,
22
  "prediction_length": 24
23
  }
 
1
  {
2
  "model": "Chronos-Bolt-Base",
3
  "api_model_id": "amazon/chronos-bolt-base",
4
+ "evaluated_at": "2026-05-28T10:12:05.182036+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "berlin/H",
8
+ "data_fetched_at": "2026-05-28T10:10:35.736508+00:00",
9
  "context_length": 504,
10
  "prediction_length": 24
11
  },
12
  {
13
  "dataset": "nyc/H",
14
+ "data_fetched_at": "2026-05-28T10:11:53.331496+00:00",
15
  "context_length": 504,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "tokyo/H",
20
+ "data_fetched_at": "2026-05-28T10:12:00.889157+00:00",
21
  "context_length": 504,
22
  "prediction_length": 24
23
  }
results/chronos_bolt_mini/all_results.csv ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ berlin/H/short,Chronos-Bolt-Mini,23.02803795462124,23.02803795462124,3.1608052717314825,0.6035438633560637,0.15491954678594283,0.1448325903393575,3.115631327236734,4.798753791831921,0.19212093626207102,0.1265447019142281,0.1081969714859019,Energy,1
3
+ nyc/H/short,Chronos-Bolt-Mini,67.67541915984299,67.67541915984299,5.3830602169036865,0.6020463455851162,0.13388074766003255,0.11707076108417061,4.7290140050230995,8.226507105682398,0.022892831572236993,0.014980050379299257,0.01192178705488713,Econ/Fin,1
4
+ tokyo/H/short,Chronos-Bolt-Mini,5.129979933399418,5.129979933399418,1.3517586653534737,0.5165812565468629,0.37651829322351843,0.8238071342871242,3.8948858261463672,2.264945900766598,0.250040044563296,0.14922819869975965,0.106032322595721,Transport,1
results/chronos_bolt_mini/config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "Chronos-Bolt-Mini",
3
+ "model_type": "zero-shot",
4
+ "model_dtype": "float32",
5
+ "model_link": "https://tsfm.ai/models/amazon/chronos-bolt-mini",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
7
+ "org": "Amazon / TSFM.ai",
8
+ "testdata_leakage": "No",
9
+ "replication_code_available": "Yes",
10
+ "api_model_id": "amazon/chronos-bolt-mini"
11
+ }
results/chronos_bolt_mini/online_meta.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "Chronos-Bolt-Mini",
3
+ "api_model_id": "amazon/chronos-bolt-mini",
4
+ "evaluated_at": "2026-05-28T10:10:33.484071+00:00",
5
+ "datasets": [
6
+ {
7
+ "dataset": "berlin/H",
8
+ "data_fetched_at": "2026-05-28T10:10:08.921860+00:00",
9
+ "context_length": 504,
10
+ "prediction_length": 24
11
+ },
12
+ {
13
+ "dataset": "nyc/H",
14
+ "data_fetched_at": "2026-05-28T10:10:20.642074+00:00",
15
+ "context_length": 504,
16
+ "prediction_length": 24
17
+ },
18
+ {
19
+ "dataset": "tokyo/H",
20
+ "data_fetched_at": "2026-05-28T10:10:28.780465+00:00",
21
+ "context_length": 504,
22
+ "prediction_length": 24
23
+ }
24
+ ]
25
+ }
results/chronos_bolt_tiny/online_meta.json CHANGED
@@ -1,23 +1,23 @@
1
  {
2
  "model": "Chronos-Bolt-Tiny",
3
  "api_model_id": "amazon/chronos-bolt-tiny",
4
- "evaluated_at": "2026-05-28T09:57:39.194711+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "berlin/H",
8
- "data_fetched_at": "2026-05-28T09:56:56.023164+00:00",
9
  "context_length": 504,
10
  "prediction_length": 24
11
  },
12
  {
13
  "dataset": "nyc/H",
14
- "data_fetched_at": "2026-05-28T09:57:26.649671+00:00",
15
  "context_length": 504,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "tokyo/H",
20
- "data_fetched_at": "2026-05-28T09:57:33.990365+00:00",
21
  "context_length": 504,
22
  "prediction_length": 24
23
  }
 
1
  {
2
  "model": "Chronos-Bolt-Tiny",
3
  "api_model_id": "amazon/chronos-bolt-tiny",
4
+ "evaluated_at": "2026-05-28T10:09:59.581604+00:00",
5
  "datasets": [
6
  {
7
  "dataset": "berlin/H",
8
+ "data_fetched_at": "2026-05-28T10:09:00.615970+00:00",
9
  "context_length": 504,
10
  "prediction_length": 24
11
  },
12
  {
13
  "dataset": "nyc/H",
14
+ "data_fetched_at": "2026-05-28T10:09:46.973725+00:00",
15
  "context_length": 504,
16
  "prediction_length": 24
17
  },
18
  {
19
  "dataset": "tokyo/H",
20
+ "data_fetched_at": "2026-05-28T10:09:54.263751+00:00",
21
  "context_length": 504,
22
  "prediction_length": 24
23
  }
results/moirai_1_1_small/all_results.csv ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ berlin/H/short,Moirai-1.1-Small,25.942115142478016,25.914808326787288,4.134999871253967,0.8981191360243419,0.2807477289705378,0.2263154350433037,4.74372356414242,5.0933402735805915,0.20391487967713806,0.16554715685998675,0.13114703596375427,Energy,1
3
+ nyc/H/short,Moirai-1.1-Small,115.85427334839227,120.1817511764708,7.214917302131653,0.797558713399391,0.19310537226504765,0.15686588488083114,4.257123747918823,10.763562298253877,0.029952981945521127,0.02007776623583399,0.014418479379084874,Econ/Fin,1
4
+ tokyo/H/short,Moirai-1.1-Small,5.042828195691895,4.650267012598303,1.3240045196531962,0.49590462426539694,0.38585838156843166,0.8294245926005037,4.263774493409405,2.2456242329677276,0.2479070175996799,0.1461642633424671,0.11333818582137908,Transport,1
results/moirai_1_1_small/config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "Moirai-1.1-Small",
3
+ "model_type": "zero-shot",
4
+ "model_dtype": "float32",
5
+ "model_link": "https://tsfm.ai/models/Salesforce/moirai-1.1-R-small",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
7
+ "org": "Salesforce / TSFM.ai",
8
+ "testdata_leakage": "No",
9
+ "replication_code_available": "Yes",
10
+ "api_model_id": "Salesforce/moirai-1.1-R-small"
11
+ }
results/moirai_1_1_small/online_meta.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "Moirai-1.1-Small",
3
+ "api_model_id": "Salesforce/moirai-1.1-R-small",
4
+ "evaluated_at": "2026-05-28T10:18:27.617821+00:00",
5
+ "datasets": [
6
+ {
7
+ "dataset": "berlin/H",
8
+ "data_fetched_at": "2026-05-28T10:18:02.944773+00:00",
9
+ "context_length": 504,
10
+ "prediction_length": 24
11
+ },
12
+ {
13
+ "dataset": "nyc/H",
14
+ "data_fetched_at": "2026-05-28T10:18:14.313951+00:00",
15
+ "context_length": 504,
16
+ "prediction_length": 24
17
+ },
18
+ {
19
+ "dataset": "tokyo/H",
20
+ "data_fetched_at": "2026-05-28T10:18:22.281838+00:00",
21
+ "context_length": 504,
22
+ "prediction_length": 24
23
+ }
24
+ ]
25
+ }
results/online_status.json CHANGED
@@ -1,85 +1,61 @@
1
  {
2
  "status": "ok",
3
- "started_at": "2026-05-28T09:56:53.592771+00:00",
4
- "finished_at": "2026-05-28T09:58:41.599812+00:00",
5
  "data_source": "open_meteo",
6
  "data_config": "configs/datasets/open_meteo.yaml",
7
  "baseline_model": "Chronos-Bolt-Base",
8
  "models": {
9
- "Chronos-Bolt-Tiny": {
10
- "model": "Chronos-Bolt-Tiny",
11
- "api_model_id": "amazon/chronos-bolt-tiny",
12
- "evaluated_at": "2026-05-28T09:57:39.194711+00:00",
13
  "datasets": [
14
  {
15
  "dataset": "berlin/H",
16
- "data_fetched_at": "2026-05-28T09:56:56.023164+00:00",
17
  "context_length": 504,
18
  "prediction_length": 24
19
  },
20
  {
21
  "dataset": "nyc/H",
22
- "data_fetched_at": "2026-05-28T09:57:26.649671+00:00",
23
  "context_length": 504,
24
  "prediction_length": 24
25
  },
26
  {
27
  "dataset": "tokyo/H",
28
- "data_fetched_at": "2026-05-28T09:57:33.990365+00:00",
29
  "context_length": 504,
30
  "prediction_length": 24
31
  }
32
  ]
33
  },
34
- "Chronos-Bolt-Base": {
35
- "model": "Chronos-Bolt-Base",
36
- "api_model_id": "amazon/chronos-bolt-base",
37
- "evaluated_at": "2026-05-28T09:58:12.886980+00:00",
38
  "datasets": [
39
  {
40
  "dataset": "berlin/H",
41
- "data_fetched_at": "2026-05-28T09:57:42.662907+00:00",
42
  "context_length": 504,
43
  "prediction_length": 24
44
  },
45
  {
46
  "dataset": "nyc/H",
47
- "data_fetched_at": "2026-05-28T09:57:56.819035+00:00",
48
  "context_length": 504,
49
  "prediction_length": 24
50
  },
51
  {
52
  "dataset": "tokyo/H",
53
- "data_fetched_at": "2026-05-28T09:58:06.485785+00:00",
54
- "context_length": 504,
55
- "prediction_length": 24
56
- }
57
- ]
58
- },
59
- "Chronos-2": {
60
- "model": "Chronos-2",
61
- "api_model_id": "amazon/chronos-2",
62
- "evaluated_at": "2026-05-28T09:58:41.598566+00:00",
63
- "datasets": [
64
- {
65
- "dataset": "berlin/H",
66
- "data_fetched_at": "2026-05-28T09:58:16.749981+00:00",
67
- "context_length": 504,
68
- "prediction_length": 24
69
- },
70
- {
71
- "dataset": "nyc/H",
72
- "data_fetched_at": "2026-05-28T09:58:28.553123+00:00",
73
- "context_length": 504,
74
- "prediction_length": 24
75
- },
76
- {
77
- "dataset": "tokyo/H",
78
- "data_fetched_at": "2026-05-28T09:58:37.331184+00:00",
79
  "context_length": 504,
80
  "prediction_length": 24
81
  }
82
  ]
83
  }
84
- }
 
85
  }
 
1
  {
2
  "status": "ok",
3
+ "started_at": "2026-05-28T10:17:24.399004+00:00",
4
+ "finished_at": "2026-05-28T10:18:27.618686+00:00",
5
  "data_source": "open_meteo",
6
  "data_config": "configs/datasets/open_meteo.yaml",
7
  "baseline_model": "Chronos-Bolt-Base",
8
  "models": {
9
+ "TimesFM-2.5": {
10
+ "model": "TimesFM-2.5",
11
+ "api_model_id": "google/timesfm-2.5-200m-pytorch",
12
+ "evaluated_at": "2026-05-28T10:18:01.120947+00:00",
13
  "datasets": [
14
  {
15
  "dataset": "berlin/H",
16
+ "data_fetched_at": "2026-05-28T10:17:26.673802+00:00",
17
  "context_length": 504,
18
  "prediction_length": 24
19
  },
20
  {
21
  "dataset": "nyc/H",
22
+ "data_fetched_at": "2026-05-28T10:17:42.004619+00:00",
23
  "context_length": 504,
24
  "prediction_length": 24
25
  },
26
  {
27
  "dataset": "tokyo/H",
28
+ "data_fetched_at": "2026-05-28T10:17:52.661277+00:00",
29
  "context_length": 504,
30
  "prediction_length": 24
31
  }
32
  ]
33
  },
34
+ "Moirai-1.1-Small": {
35
+ "model": "Moirai-1.1-Small",
36
+ "api_model_id": "Salesforce/moirai-1.1-R-small",
37
+ "evaluated_at": "2026-05-28T10:18:27.617821+00:00",
38
  "datasets": [
39
  {
40
  "dataset": "berlin/H",
41
+ "data_fetched_at": "2026-05-28T10:18:02.944773+00:00",
42
  "context_length": 504,
43
  "prediction_length": 24
44
  },
45
  {
46
  "dataset": "nyc/H",
47
+ "data_fetched_at": "2026-05-28T10:18:14.313951+00:00",
48
  "context_length": 504,
49
  "prediction_length": 24
50
  },
51
  {
52
  "dataset": "tokyo/H",
53
+ "data_fetched_at": "2026-05-28T10:18:22.281838+00:00",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
  "context_length": 504,
55
  "prediction_length": 24
56
  }
57
  ]
58
  }
59
+ },
60
+ "failed_models": []
61
  }
results/timesfm_2_5/all_results.csv ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates
2
+ berlin/H/short,TimesFM-2.5,26.54830902205753,31.322212017679387,3.443761388460795,0.6303993503146956,0.1386495615012084,0.13621565370038174,18.01169611505954,5.152505121012257,0.20628357921361393,0.13787301681124278,0.1312424613907817,Energy,1
3
+ nyc/H/short,TimesFM-2.5,66.51865488618301,44.39263497040171,4.487805022133721,0.4947887756747412,0.09551055813760116,0.08531003986906531,15.677110160428015,8.155896939404213,0.02269633667797287,0.012488722513809137,0.014150956333775139,Econ/Fin,1
4
+ tokyo/H/short,TimesFM-2.5,4.1984207631854,2.872721300592553,1.081747318436909,0.411514142106336,0.31230229584942987,0.6226950610386188,18.191892843171864,2.0490048226359545,0.2262011013105863,0.11942013609095183,0.13494832121177208,Transport,1
results/timesfm_2_5/config.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "TimesFM-2.5",
3
+ "model_type": "zero-shot",
4
+ "model_dtype": "float32",
5
+ "model_link": "https://tsfm.ai/models/google/timesfm-2.5-200m-pytorch",
6
+ "code_link": "https://github.com/zhouziyu02/TSFM_Bench/blob/main/scripts/run_online_eval.py",
7
+ "org": "Google / TSFM.ai",
8
+ "testdata_leakage": "No",
9
+ "replication_code_available": "Yes",
10
+ "api_model_id": "google/timesfm-2.5-200m-pytorch"
11
+ }
results/timesfm_2_5/online_meta.json ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "TimesFM-2.5",
3
+ "api_model_id": "google/timesfm-2.5-200m-pytorch",
4
+ "evaluated_at": "2026-05-28T10:18:01.120947+00:00",
5
+ "datasets": [
6
+ {
7
+ "dataset": "berlin/H",
8
+ "data_fetched_at": "2026-05-28T10:17:26.673802+00:00",
9
+ "context_length": 504,
10
+ "prediction_length": 24
11
+ },
12
+ {
13
+ "dataset": "nyc/H",
14
+ "data_fetched_at": "2026-05-28T10:17:42.004619+00:00",
15
+ "context_length": 504,
16
+ "prediction_length": 24
17
+ },
18
+ {
19
+ "dataset": "tokyo/H",
20
+ "data_fetched_at": "2026-05-28T10:17:52.661277+00:00",
21
+ "context_length": 504,
22
+ "prediction_length": 24
23
+ }
24
+ ]
25
+ }
src/about.py CHANGED
@@ -3,7 +3,9 @@ TITLE = """ TSFM Realworld Bench Leaderboard """
3
  INTRODUCTION_TEXT = """
4
  **TSFM Realworld Bench** evaluates time series foundation models on **live Open-Meteo
5
  weather data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
6
- Scores are normalized against **Chronos-Bolt-Base**; lower is better.
 
 
7
  """
8
 
9
  LLM_BENCHMARKS_TEXT = """
 
3
  INTRODUCTION_TEXT = """
4
  **TSFM Realworld Bench** evaluates time series foundation models on **live Open-Meteo
5
  weather data** with **zero-shot API inference** via [TSFM.ai](https://tsfm.ai/).
6
+ Scores are normalized against **Chronos-Bolt-Base**. The leaderboard shows
7
+ **metric values** and **ranks** in separate tables (MASE, CRPS, MAE, RMSE, sMAPE,
8
+ MSIS, ND, NRMSE, MAPE).
9
  """
10
 
11
  LLM_BENCHMARKS_TEXT = """
src/utils.py CHANGED
@@ -6,8 +6,6 @@ from pathlib import Path
6
  import pandas as pd
7
  from scipy import stats
8
 
9
- from src.leaderboard.read_evals import get_model_info
10
-
11
  METRIC_COLUMNS = [
12
  "eval_metrics/MSE[mean]",
13
  "eval_metrics/MSE[0.5]",
@@ -21,9 +19,33 @@ METRIC_COLUMNS = [
21
  "eval_metrics/ND[0.5]",
22
  "eval_metrics/mean_weighted_sum_quantile_loss",
23
  ]
24
- RANKING_METRIC = "eval_metrics/mean_weighted_sum_quantile_loss"
25
- MASE_RANKING_METRIC = "eval_metrics/MASE[0.5]"
26
- METRIC_CHOICES = [MASE_RANKING_METRIC, RANKING_METRIC]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
 
28
  LEGACY_MODEL_NAMES = {
29
  "TSFM1": "Chronos-Bolt-Tiny",
@@ -49,17 +71,6 @@ def format_df(df: pd.DataFrame) -> pd.DataFrame:
49
  return formatted
50
 
51
 
52
- def rename_metrics(df: pd.DataFrame) -> pd.DataFrame:
53
- return df.rename(
54
- columns={
55
- "eval_metrics/MASE[0.5]": "MASE",
56
- "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS",
57
- "rank": "CRPS_Rank",
58
- "Rank_MASE": "MASE_Rank",
59
- }
60
- )
61
-
62
-
63
  def unify_freq(df: pd.DataFrame) -> pd.DataFrame:
64
  df = df.copy()
65
  df["frequency"] = df["frequency"].str.replace(r"\d+", "", regex=True)
@@ -88,7 +99,8 @@ def standardize_df(df: pd.DataFrame, root_dir: str = "results") -> pd.DataFrame:
88
  if baseline_name not in set(df["model"]):
89
  return df
90
 
91
- for _, group in df.groupby(["dataset", "term_length", "frequency"]):
 
92
  baseline = group[group["model"] == baseline_name]
93
  if baseline.empty:
94
  continue
@@ -131,46 +143,51 @@ def enrich_with_properties(df: pd.DataFrame, ds_properties_path: str) -> pd.Data
131
  return unify_freq(df)
132
 
133
 
 
 
 
 
 
 
 
 
 
134
  def get_grouped_dfs(
135
  root_dir: str = "results",
136
  ds_properties: str | None = None,
137
  ) -> dict[str, pd.DataFrame]:
138
  if ds_properties is None:
139
  ds_properties = str(Path(root_dir) / "dataset_properties.csv")
 
140
  df = load_all_results(root_dir)
141
  df = enrich_with_properties(df, ds_properties)
142
  df = standardize_df(df, root_dir)
 
143
 
144
- df["rank"] = df.groupby(["dataset", "term_length", "frequency"])[RANKING_METRIC].rank(
145
- method="first", ascending=True
 
 
 
 
 
146
  )
147
- df["Rank_MASE"] = df.groupby(["dataset", "term_length", "frequency"])[
148
- MASE_RANKING_METRIC
149
- ].rank(method="first", ascending=True)
150
-
151
- overall = df.groupby(["model"])[METRIC_CHOICES].agg(stats.gmean)
152
- overall_rank = df.groupby(["model"])[["rank", "Rank_MASE"]].mean()
153
- overall = pd.concat([overall, overall_rank], axis=1)
154
- overall = overall[
155
- [
156
- "eval_metrics/MASE[0.5]",
157
- "Rank_MASE",
158
- "eval_metrics/mean_weighted_sum_quantile_loss",
159
- "rank",
160
- ]
161
- ]
162
-
163
- return {"overall": overall}
164
-
165
-
166
- def get_merged_df(result_df: pd.DataFrame, model_info_df: pd.DataFrame) -> pd.DataFrame:
167
- merged = pd.merge(model_info_df, result_df, left_on="model", right_on="model", how="inner")
168
- merged = merged.drop(columns=["model"])
169
- merged = merged.rename(columns={"model_w_link": "Model"})
170
- return merged
171
-
172
-
173
- def prepare_leaderboard_table(grouped_df: pd.DataFrame) -> pd.DataFrame:
174
- table = rename_metrics(grouped_df.reset_index())
175
  table = format_df(table)
176
- return table.sort_values(by=["MASE_Rank"])
 
 
6
  import pandas as pd
7
  from scipy import stats
8
 
 
 
9
  METRIC_COLUMNS = [
10
  "eval_metrics/MSE[mean]",
11
  "eval_metrics/MSE[0.5]",
 
19
  "eval_metrics/ND[0.5]",
20
  "eval_metrics/mean_weighted_sum_quantile_loss",
21
  ]
22
+
23
+ DISPLAY_METRICS = [
24
+ "eval_metrics/MASE[0.5]",
25
+ "eval_metrics/mean_weighted_sum_quantile_loss",
26
+ "eval_metrics/MAE[0.5]",
27
+ "eval_metrics/RMSE[mean]",
28
+ "eval_metrics/sMAPE[0.5]",
29
+ "eval_metrics/MSIS",
30
+ "eval_metrics/ND[0.5]",
31
+ "eval_metrics/NRMSE[mean]",
32
+ "eval_metrics/MAPE[0.5]",
33
+ ]
34
+
35
+ METRIC_LABELS = {
36
+ "eval_metrics/MASE[0.5]": "MASE",
37
+ "eval_metrics/mean_weighted_sum_quantile_loss": "CRPS",
38
+ "eval_metrics/MAE[0.5]": "MAE",
39
+ "eval_metrics/RMSE[mean]": "RMSE",
40
+ "eval_metrics/sMAPE[0.5]": "sMAPE",
41
+ "eval_metrics/MSIS": "MSIS",
42
+ "eval_metrics/ND[0.5]": "ND",
43
+ "eval_metrics/NRMSE[mean]": "NRMSE",
44
+ "eval_metrics/MAPE[0.5]": "MAPE",
45
+ }
46
+
47
+ VALUE_COLUMNS = [METRIC_LABELS[m] for m in DISPLAY_METRICS]
48
+ RANK_COLUMNS = [f"{METRIC_LABELS[m]}_Rank" for m in DISPLAY_METRICS]
49
 
50
  LEGACY_MODEL_NAMES = {
51
  "TSFM1": "Chronos-Bolt-Tiny",
 
71
  return formatted
72
 
73
 
 
 
 
 
 
 
 
 
 
 
 
74
  def unify_freq(df: pd.DataFrame) -> pd.DataFrame:
75
  df = df.copy()
76
  df["frequency"] = df["frequency"].str.replace(r"\d+", "", regex=True)
 
99
  if baseline_name not in set(df["model"]):
100
  return df
101
 
102
+ group_keys = ["dataset", "term_length", "frequency"]
103
+ for _, group in df.groupby(group_keys):
104
  baseline = group[group["model"] == baseline_name]
105
  if baseline.empty:
106
  continue
 
143
  return unify_freq(df)
144
 
145
 
146
+ def _add_per_dataset_ranks(df: pd.DataFrame) -> pd.DataFrame:
147
+ df = df.copy()
148
+ group_keys = ["dataset", "term_length", "frequency"]
149
+ for metric in DISPLAY_METRICS:
150
+ rank_col = f"Rank_{metric}"
151
+ df[rank_col] = df.groupby(group_keys)[metric].rank(method="first", ascending=True)
152
+ return df
153
+
154
+
155
  def get_grouped_dfs(
156
  root_dir: str = "results",
157
  ds_properties: str | None = None,
158
  ) -> dict[str, pd.DataFrame]:
159
  if ds_properties is None:
160
  ds_properties = str(Path(root_dir) / "dataset_properties.csv")
161
+
162
  df = load_all_results(root_dir)
163
  df = enrich_with_properties(df, ds_properties)
164
  df = standardize_df(df, root_dir)
165
+ df = _add_per_dataset_ranks(df)
166
 
167
+ rank_cols = [f"Rank_{metric}" for metric in DISPLAY_METRICS]
168
+ overall_values = df.groupby(["model"])[DISPLAY_METRICS].agg(stats.gmean)
169
+ overall_ranks = df.groupby(["model"])[rank_cols].mean()
170
+
171
+ overall_values = overall_values.rename(columns=METRIC_LABELS)
172
+ overall_ranks = overall_ranks.rename(
173
+ columns={f"Rank_{metric}": f"{METRIC_LABELS[metric]}_Rank" for metric in DISPLAY_METRICS}
174
  )
175
+
176
+ return {
177
+ "overall_values": overall_values,
178
+ "overall_ranks": overall_ranks,
179
+ }
180
+
181
+
182
+ def prepare_values_table(values_df: pd.DataFrame) -> pd.DataFrame:
183
+ table = values_df.reset_index().rename(columns={"model": "Model"})
184
+ table = format_df(table)
185
+ cols = ["Model"] + [col for col in VALUE_COLUMNS if col in table.columns]
186
+ return table[cols].sort_values(by=["MASE"])
187
+
188
+
189
+ def prepare_ranks_table(ranks_df: pd.DataFrame) -> pd.DataFrame:
190
+ table = ranks_df.reset_index().rename(columns={"model": "Model"})
 
 
 
 
 
 
 
 
 
 
 
 
191
  table = format_df(table)
192
+ cols = ["Model"] + [col for col in RANK_COLUMNS if col in table.columns]
193
+ return table[cols].sort_values(by=["MASE_Rank"])