update leaderboard

README.md

@@ -40,14 +40,21 @@ Required fields per record:
   "T2_acc": 0.0,
   "T3_acc": 0.0,
   "T4_acc": 0.0,
+  "T2_sMAPE": 0.0,
   "T2_MAE": 0.0,
   "T4_sMAPE": 0.0,
-  "Retail_T3_acc": 0.0
+  "T4_MAE": 0.0,
+  "FreshRetailNet_T2_sMAPE": 0.0,
+  "FreshRetailNet_T2_MAE": 0.0,
+  "MIMIC_T2_OW_sMAPE": 0.0,
+  "MIMIC_T2_OW_RMSSE": 0.0
 }
 ```
 
 Notes:
-- `T2_MAE` and `T4_sMAPE` are optional.
+- `T2_sMAPE`, `T2_MAE`, `T4_sMAPE`, `T4_MAE` are optional (forecasting metrics).
+- Dataset-level columns are optional and displayed if present.
+- For MIMIC forecasting, only `OW_sMAPE` and `OW_RMSSE` are expected.
 - Any additional numeric columns are treated as optional domain metrics and will be shown.
 - Records must have a consistent schema and numeric metric values.
 

app.py

@@ -37,16 +37,41 @@ def load_leaderboard_data() -> tuple[pd.DataFrame, list[str], Optional[str]]:
 
 
 LEADERBOARD_DF, COLUMN_ORDER, LOAD_ERROR = load_leaderboard_data()
-METRIC_COLUMNS = [c for c in COLUMN_ORDER if c not in SCHEMA.identity_fields]
 
-COMPARE_OPTIONS = []
-COMPARE_LOOKUP = {}
-for idx, row in LEADERBOARD_DF.iterrows():
-    label = (
-        f"{row['agent_name']} | {row['model_name']} | {row['agent_type']} | {row['base_model']} ({idx})"
-    )
-    COMPARE_OPTIONS.append(label)
-    COMPARE_LOOKUP[label] = row.to_dict()
+DATASET_DISPLAY_NAMES = ["FreshRetailNet", "PSML", "Causal Chambers", "MIMIC"]
+DATASET_PREFIX_MAP = {
+    "FreshRetailNet": "FreshRetailNet",
+    "PSML": "PSML",
+    "Causal Chambers": "CausalChambers",
+    "MIMIC": "MIMIC",
+}
+DATASET_PREFIXES = [f"{prefix}_" for prefix in DATASET_PREFIX_MAP.values()]
+
+
+def is_dataset_metric(column: str) -> bool:
+    return any(column.startswith(prefix) for prefix in DATASET_PREFIXES)
+
+
+BASE_COLUMNS = list(SCHEMA.identity_fields) + list(SCHEMA.required_metrics)
+ALL_DATASET_COLUMNS = [c for c in COLUMN_ORDER if is_dataset_metric(c)]
+
+AGGREGATE_FORECAST_COLUMNS = [
+    "overall_mcq_acc",
+    "T2_MAE",
+    "T2_sMAPE",
+    "T4_MAE",
+    "T4_sMAPE",
+    "MIMIC_T2_OW_sMAPE",
+    "MIMIC_T2_OW_RMSSE",
+    "MIMIC_T4_OW_sMAPE",
+    "MIMIC_T4_OW_RMSSE",
+]
+AGGREGATE_COLUMNS = BASE_COLUMNS + [
+    c for c in AGGREGATE_FORECAST_COLUMNS if c in COLUMN_ORDER
+]
+
+DISPLAY_ALL_COLUMNS = BASE_COLUMNS + ALL_DATASET_COLUMNS
+BY_DOMAIN_COLUMNS = BASE_COLUMNS + ALL_DATASET_COLUMNS
 
 
 def column_types(column_order: list[str]) -> list[str]:
@@ -62,10 +87,13 @@ def column_types(column_order: list[str]) -> list[str]:
 def init_leaderboard(dataframe, column_order):
     if dataframe is None or dataframe.empty:
         dataframe = pd.DataFrame(columns=column_order)
+    dataframe = dataframe.reindex(columns=column_order)
 
     required_cols = list(SCHEMA.identity_fields) + list(SCHEMA.required_metrics)
     cant_deselect = [c for c in required_cols if c in column_order]
 
+    search_columns = [c for c in ["model_name", "agent_name"] if c in column_order]
+
     return Leaderboard(
         value=dataframe,
         datatype=column_types(column_order),
@@ -74,7 +102,7 @@ def init_leaderboard(dataframe, column_order):
             cant_deselect=cant_deselect,
             label="Select Columns to Display:",
         ),
-        search_columns=["model_name", "agent_name"],
+        search_columns=search_columns,
         filter_columns=[
             ColumnFilter("agent_type", type="checkboxgroup", label="Agent type"),
         ],
@@ -82,30 +110,7 @@ def init_leaderboard(dataframe, column_order):
     )
 
 
-def compare_entries(entry_a: str, entry_b: str) -> pd.DataFrame:
-    if not entry_a or not entry_b:
-        return pd.DataFrame(columns=["metric", "entry_a", "entry_b", "delta"])
-    row_a = COMPARE_LOOKUP.get(entry_a)
-    row_b = COMPARE_LOOKUP.get(entry_b)
-    if row_a is None or row_b is None:
-        return pd.DataFrame(columns=["metric", "entry_a", "entry_b", "delta"])
-
-    rows = []
-    for metric in METRIC_COLUMNS:
-        value_a = row_a.get(metric)
-        value_b = row_b.get(metric)
-        delta = None
-        if value_a is not None and value_b is not None:
-            delta = value_b - value_a
-        rows.append(
-            {
-                "metric": metric,
-                "entry_a": value_a,
-                "entry_b": value_b,
-                "delta": delta,
-            }
-        )
-    return pd.DataFrame.from_records(rows)
+ 
 
 
 def save_submission(uploaded_file) -> str:
@@ -134,6 +139,17 @@ def save_submission(uploaded_file) -> str:
     return f"Submission received for review. Saved to `{out_path}`."
 
 
+def example_record_markdown() -> str:
+    try:
+        records = load_records(RESULTS_PATH)
+        if not records:
+            return "No example data available."
+        example = records[0]
+        return "Example record (JSON):\n```json\n" + json.dumps(example, indent=2) + "\n```"
+    except Exception as exc:
+        return f"Could not load example record: {exc}"
+
+
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.HTML(TITLE)
@@ -143,31 +159,18 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF, COLUMN_ORDER)
+            leaderboard = init_leaderboard(LEADERBOARD_DF, AGGREGATE_COLUMNS)
 
-        with gr.TabItem("🔍 Compare", elem_id="llm-benchmark-tab-table", id=1):
-            gr.Markdown(
-                "Select two evaluated entries to compare their metrics side by side.",
-                elem_classes="markdown-text",
-            )
-            with gr.Row():
-                entry_a = gr.Dropdown(choices=COMPARE_OPTIONS, label="Entry A", value=None)
-                entry_b = gr.Dropdown(choices=COMPARE_OPTIONS, label="Entry B", value=None)
-            compare_table = gr.Dataframe(
-                value=pd.DataFrame(columns=["metric", "entry_a", "entry_b", "delta"]),
-                headers=["metric", "entry_a", "entry_b", "delta"],
-                datatype=["str", "number", "number", "number"],
-                interactive=False,
-                row_count=10,
-            )
-            entry_a.change(compare_entries, [entry_a, entry_b], compare_table)
-            entry_b.change(compare_entries, [entry_a, entry_b], compare_table)
+        with gr.TabItem("🧭 By Domain", elem_id="llm-benchmark-tab-table", id=1):
+            by_domain_df = LEADERBOARD_DF.reindex(columns=BY_DOMAIN_COLUMNS)
+            init_leaderboard(by_domain_df, BY_DOMAIN_COLUMNS)
 
         with gr.TabItem("📤 Submit Results", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(
                 "Upload a results file for manual review. Approved results will be merged into the main dataset.",
                 elem_classes="markdown-text",
             )
+            gr.Markdown(example_record_markdown(), elem_classes="markdown-text")
             submission_file = gr.File(label="Results file (.json or .csv)", file_types=[".json", ".csv"])
             submit_button = gr.Button("Submit for Review")
             submission_status = gr.Markdown()
@@ -176,14 +179,15 @@ with demo:
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=3):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-    with gr.Row():
-        with gr.Accordion("📙 Citation", open=False):
-            citation_button = gr.Textbox(
-                value=CITATION_BUTTON_TEXT,
-                label=CITATION_BUTTON_LABEL,
-                lines=20,
-                elem_id="citation-button",
-                show_copy_button=True,
-            )
+    # Citation section hidden for now.
+    # with gr.Row():
+    #     with gr.Accordion("📙 Citation", open=False):
+    #         citation_button = gr.Textbox(
+    #             value=CITATION_BUTTON_TEXT,
+    #             label=CITATION_BUTTON_LABEL,
+    #             lines=20,
+    #             elem_id="citation-button",
+    #             show_copy_button=True,
+    #         )
 
 demo.queue(default_concurrency_limit=40).launch()

data/results.json

@@ -1,41 +1,247 @@
 [
   {
-    "model_name": "demo-model-1",
-    "agent_name": "TemporalAgent-A",
+    "agent_name": "Single LLM",
     "agent_type": "single-LLM",
-    "base_model": "demo-base-1",
-    "T1_acc": 71.2,
-    "T2_acc": 64.5,
-    "T3_acc": 69.8,
-    "T4_acc": 62.3,
-    "T2_MAE": 0.41,
-    "T4_sMAPE": 0.22,
-    "Retail_T3_acc": 70.1
+    "base_model": "gpt-4o",
+    "T1_acc": null,
+    "T2_acc": null,
+    "T3_acc": null,
+    "T4_acc": null,
+    "FreshRetailNet_T1_acc": 0.6364,
+    "FreshRetailNet_T2_acc": 0.5227,
+    "FreshRetailNet_T3_acc": 0.0289,
+    "FreshRetailNet_T4_acc": 0.1364,
+    "PSML_T1_acc": 0.675,
+    "PSML_T2_acc": 0.2067,
+    "PSML_T3_acc": 0.348,
+    "PSML_T4_acc": 0.36,
+    "CausalChambers_T1_acc": 0.1333,
+    "CausalChambers_T2_acc": 0.2733,
+    "CausalChambers_T3_acc": 0.352,
+    "CausalChambers_T4_acc": 0.26,
+    "MIMIC_T1_acc": 0.4681,
+    "MIMIC_T2_acc": 0.2128,
+    "MIMIC_T3_acc": 0.3661,
+    "MIMIC_T4_acc": 0.2979,
+    "T2_sMAPE": null,
+    "T2_MAE": null,
+    "T2_OW_sMAPE_MIMIC": null,
+    "T2_OW_RMSSE_MIMIC": null,
+    "T4_sMAPE": null,
+    "T4_MAE": null,
+    "T4_OW_sMAPE_MIMIC": null,
+    "T4_OW_RMSSE_MIMIC": null,
+    "FreshRetailNet_T2_MAE": 0.12,
+    "FreshRetailNet_T2_sMAPE": 1.27,
+    "FreshRetailNet_T4_MAE": 0.34,
+    "FreshRetailNet_T4_sMAPE": 1.29,
+    "PSML_T2_MAE": 0.61,
+    "PSML_T2_sMAPE": 0.6,
+    "PSML_T4_MAE": 0.44,
+    "PSML_T4_sMAPE": 0.37,
+    "CausalChambers_T2_MAE": 2.48,
+    "CausalChambers_T2_OW_RMSSE": 0.0000257,
+    "CausalChambers_T4_MAE": 2.58,
+    "CausalChambers_T4_OW_RMSSE": 0.0000269,
+    "MIMIC_T2_OW_sMAPE": 15.2,
+    "MIMIC_T2_OW_RMSSE": 0.55,
+    "MIMIC_T4_OW_sMAPE": 16.86,
+    "MIMIC_T4_OW_RMSSE": 0.63
   },
   {
-    "model_name": "demo-model-2",
-    "agent_name": "TemporalAgent-B",
+    "agent_name": "TimeSeries Scientist",
+    "agent_type": "time-series-specific agent",
+    "base_model": "gpt-4o",
+    "T1_acc": null,
+    "T2_acc": null,
+    "T3_acc": null,
+    "T4_acc": null,
+    "FreshRetailNet_T1_acc": 0.3352,
+    "FreshRetailNet_T2_acc": 0.5682,
+    "FreshRetailNet_T3_acc": 0.0341,
+    "FreshRetailNet_T4_acc": 0.5682,
+    "PSML_T1_acc": 0.28,
+    "PSML_T2_acc": 0.2667,
+    "PSML_T3_acc": 0.216,
+    "PSML_T4_acc": 0.2733,
+    "CausalChambers_T1_acc": 0.2867,
+    "CausalChambers_T2_acc": 0.0267,
+    "CausalChambers_T3_acc": 0.216,
+    "CausalChambers_T4_acc": 0.0267,
+    "MIMIC_T1_acc": 0.1011,
+    "MIMIC_T2_acc": 0.234,
+    "MIMIC_T3_acc": 0.2887,
+    "MIMIC_T4_acc": 0.234,
+    "T2_sMAPE": null,
+    "T2_MAE": null,
+    "T2_OW_sMAPE_MIMIC": null,
+    "T2_OW_RMSSE_MIMIC": null,
+    "T4_sMAPE": null,
+    "T4_MAE": null,
+    "T4_OW_sMAPE_MIMIC": null,
+    "T4_OW_RMSSE_MIMIC": null,
+    "FreshRetailNet_T2_MAE": 0.35,
+    "FreshRetailNet_T2_sMAPE": 1.27,
+    "FreshRetailNet_T4_MAE": 0.51,
+    "FreshRetailNet_T4_sMAPE": 1.4,
+    "PSML_T2_MAE": 1.53,
+    "PSML_T2_sMAPE": 0.65,
+    "PSML_T4_MAE": 0.84,
+    "PSML_T4_sMAPE": 0.48,
+    "CausalChambers_T2_MAE": 2.44,
+    "CausalChambers_T2_OW_RMSSE": 0.0000253,
+    "CausalChambers_T4_MAE": 2.94,
+    "CausalChambers_T4_OW_RMSSE": 0.0000306,
+    "MIMIC_T2_OW_sMAPE": 15.81,
+    "MIMIC_T2_OW_RMSSE": 0.52,
+    "MIMIC_T4_OW_sMAPE": 17.18,
+    "MIMIC_T4_OW_RMSSE": 0.64
+  },
+  {
+    "agent_name": "AgentScope",
     "agent_type": "general agent",
-    "base_model": "demo-base-2",
-    "T1_acc": 75.4,
-    "T2_acc": 66.7,
-    "T3_acc": 72.9,
-    "T4_acc": 65.8,
-    "T2_MAE": 0.38,
-    "T4_sMAPE": 0.20,
-    "MIMIC_T3_acc": 71.6
+    "base_model": "gpt-4o",
+    "T1_acc": null,
+    "T2_acc": null,
+    "T3_acc": null,
+    "T4_acc": null,
+    "FreshRetailNet_T1_acc": 0.625,
+    "FreshRetailNet_T2_acc": 0.1212,
+    "FreshRetailNet_T3_acc": 0.1364,
+    "FreshRetailNet_T4_acc": 0.1894,
+    "PSML_T1_acc": 0.66,
+    "PSML_T2_acc": 0.2467,
+    "PSML_T3_acc": 0.272,
+    "PSML_T4_acc": 0.3533,
+    "CausalChambers_T1_acc": 0.12,
+    "CausalChambers_T2_acc": 0.46,
+    "CausalChambers_T3_acc": 0.44,
+    "CausalChambers_T4_acc": 0.32,
+    "MIMIC_T1_acc": 0.4468,
+    "MIMIC_T2_acc": 0.2128,
+    "MIMIC_T3_acc": 0.2395,
+    "MIMIC_T4_acc": 0.227,
+    "T2_sMAPE": null,
+    "T2_MAE": null,
+    "T2_OW_sMAPE_MIMIC": null,
+    "T2_OW_RMSSE_MIMIC": null,
+    "T4_sMAPE": null,
+    "T4_MAE": null,
+    "T4_OW_sMAPE_MIMIC": null,
+    "T4_OW_RMSSE_MIMIC": null,
+    "FreshRetailNet_T2_MAE": 0.12,
+    "FreshRetailNet_T2_sMAPE": 126.27,
+    "FreshRetailNet_T4_MAE": 0.2,
+    "FreshRetailNet_T4_sMAPE": 130.86,
+    "PSML_T2_MAE": 0.28,
+    "PSML_T2_sMAPE": 37.38,
+    "PSML_T4_MAE": 0.35,
+    "PSML_T4_sMAPE": 30.51,
+    "CausalChambers_T2_MAE": 2.76,
+    "CausalChambers_T2_OW_RMSSE": 0.00262,
+    "CausalChambers_T4_MAE": 2.66,
+    "CausalChambers_T4_OW_RMSSE": 0.00246,
+    "MIMIC_T2_OW_sMAPE": 11.05,
+    "MIMIC_T2_OW_RMSSE": 0.43,
+    "MIMIC_T4_OW_sMAPE": 12.02,
+    "MIMIC_T4_OW_RMSSE": 0.49
   },
   {
-    "model_name": "demo-model-3",
-    "agent_name": "TemporalAgent-C",
-    "agent_type": "time-series-specific agent",
-    "base_model": "demo-base-3",
-    "T1_acc": 69.9,
-    "T2_acc": 63.2,
-    "T3_acc": 68.4,
-    "T4_acc": 61.7,
-    "T2_MAE": 0.44,
-    "T4_sMAPE": 0.24,
-    "PSML_T3_acc": 67.9
+    "agent_name": "MetaGPT",
+    "agent_type": "general agent",
+    "base_model": "gpt-4o",
+    "T1_acc": null,
+    "T2_acc": null,
+    "T3_acc": null,
+    "T4_acc": null,
+    "FreshRetailNet_T1_acc": 0.625,
+    "FreshRetailNet_T2_acc": 0.0909,
+    "FreshRetailNet_T3_acc": 0.0511,
+    "FreshRetailNet_T4_acc": 0.1439,
+    "PSML_T1_acc": 0.675,
+    "PSML_T2_acc": 0.2109,
+    "PSML_T3_acc": 0.22,
+    "PSML_T4_acc": 0.3133,
+    "CausalChambers_T1_acc": 0.1067,
+    "CausalChambers_T2_acc": 0.5933,
+    "CausalChambers_T3_acc": 0.452,
+    "CausalChambers_T4_acc": 0.16,
+    "MIMIC_T1_acc": 0.4574,
+    "MIMIC_T2_acc": 0.1702,
+    "MIMIC_T3_acc": 0.2897,
+    "MIMIC_T4_acc": 0.2553,
+    "T2_sMAPE": null,
+    "T2_MAE": null,
+    "T2_OW_sMAPE_MIMIC": null,
+    "T2_OW_RMSSE_MIMIC": null,
+    "T4_sMAPE": null,
+    "T4_MAE": null,
+    "T4_OW_sMAPE_MIMIC": null,
+    "T4_OW_RMSSE_MIMIC": null,
+    "FreshRetailNet_T2_MAE": 0.13,
+    "FreshRetailNet_T2_sMAPE": 126.59,
+    "FreshRetailNet_T4_MAE": 0.24,
+    "FreshRetailNet_T4_sMAPE": 127.22,
+    "PSML_T2_MAE": 0.34,
+    "PSML_T2_sMAPE": 24.74,
+    "PSML_T4_MAE": 0.4,
+    "PSML_T4_sMAPE": 43.47,
+    "CausalChambers_T2_MAE": 2.62,
+    "CausalChambers_T2_OW_RMSSE": 0.00272,
+    "CausalChambers_T4_MAE": 2.76,
+    "CausalChambers_T4_OW_RMSSE": 0.00287,
+    "MIMIC_T2_OW_sMAPE": 14.11,
+    "MIMIC_T2_OW_RMSSE": 0.53,
+    "MIMIC_T4_OW_sMAPE": 15.4,
+    "MIMIC_T4_OW_RMSSE": 0.63
+  },
+  {
+    "agent_name": "CAMEL",
+    "agent_type": "general agent",
+    "base_model": "gpt-4o",
+    "T1_acc": null,
+    "T2_acc": null,
+    "T3_acc": null,
+    "T4_acc": null,
+    "FreshRetailNet_T1_acc": 0.642,
+    "FreshRetailNet_T2_acc": 0.0076,
+    "FreshRetailNet_T3_acc": 0.0625,
+    "FreshRetailNet_T4_acc": 0.3106,
+    "PSML_T1_acc": 0.685,
+    "PSML_T2_acc": 0.14,
+    "PSML_T3_acc": 0.184,
+    "PSML_T4_acc": 0.3067,
+    "CausalChambers_T1_acc": 0.1,
+    "CausalChambers_T2_acc": 0.66,
+    "CausalChambers_T3_acc": 0.42,
+    "CausalChambers_T4_acc": 0.2667,
+    "MIMIC_T1_acc": 0.4681,
+    "MIMIC_T2_acc": 0.2057,
+    "MIMIC_T3_acc": 0.3014,
+    "MIMIC_T4_acc": 0.234,
+    "T2_sMAPE": null,
+    "T2_MAE": null,
+    "T2_OW_sMAPE_MIMIC": null,
+    "T2_OW_RMSSE_MIMIC": null,
+    "T4_sMAPE": null,
+    "T4_MAE": null,
+    "T4_OW_sMAPE_MIMIC": null,
+    "T4_OW_RMSSE_MIMIC": null,
+    "FreshRetailNet_T2_MAE": 0.13,
+    "FreshRetailNet_T2_sMAPE": 126.75,
+    "FreshRetailNet_T4_MAE": 0.28,
+    "FreshRetailNet_T4_sMAPE": 128.18,
+    "PSML_T2_MAE": 0.43,
+    "PSML_T2_sMAPE": 34.89,
+    "PSML_T4_MAE": 0.45,
+    "PSML_T4_sMAPE": 35.78,
+    "CausalChambers_T2_MAE": 2.99,
+    "CausalChambers_T2_OW_RMSSE": 0.00311,
+    "CausalChambers_T4_MAE": 2.5,
+    "CausalChambers_T4_OW_RMSSE": 0.0026,
+    "MIMIC_T2_OW_sMAPE": 12.02,
+    "MIMIC_T2_OW_RMSSE": 0.55,
+    "MIMIC_T4_OW_sMAPE": 15.74,
+    "MIMIC_T4_OW_RMSSE": 0.59
   }
 ]

src/about.py

@@ -10,21 +10,39 @@ LLM_BENCHMARKS_TEXT = """
 ## What this leaderboard shows
 
 - One row per evaluated agent configuration
-- Task-family metrics for TemporalBench (T1–T4)
-- Optional domain-level metrics when provided (e.g., Retail_T3_acc)
+- Task-family MCQ metrics for TemporalBench (T1–T4)
+- Forecasting metrics for T2/T4 (sMAPE, MAE) and MIMIC OW metrics when provided
+- Dataset-level results for: FreshRetailNet, PSML, Causal Chambers, MIMIC
 
 ## Data requirements
 
 Results are loaded from a local JSON or CSV file. Each record must include:
 
-- Identity fields: `model_name`, `agent_name`, `agent_type`, `base_model`
-- Required metrics: `T1_acc`, `T2_acc`, `T3_acc`, `T4_acc`
-- Optional metrics: `T2_MAE`, `T4_sMAPE`, and any additional numeric columns
+- Identity fields: `agent_name`, `agent_type`, `base_model`
+- Required metrics: `T1_acc`, `T2_acc`, `T3_acc`, `T4_acc` (computed overall)
+- Optional metrics:
+  - Overall forecasting: `T2_sMAPE`, `T2_MAE`, `T4_sMAPE`, `T4_MAE`
+  - MIMIC overall OW: `MIMIC_T2_OW_sMAPE`, `MIMIC_T2_OW_RMSSE`, `MIMIC_T4_OW_sMAPE`, `MIMIC_T4_OW_RMSSE`
+  - Dataset-level metrics: `<Dataset>_T{1..4}_acc` and forecasting metrics per dataset
+
+## Overall computation
+
+Overall T1–T4 accuracy and T2/T4 forecasting metrics are computed as weighted averages
+from dataset-level results using question/series counts. Missing values are ignored.
 
 ## Submission workflow
 
 Uploads are stored locally for manual review. Approved results should be merged into
 the main results file to appear on the leaderboard.
+
+## Data access
+
+The dataset is available at:
+```
+https://huggingface.co/datasets/Melady/TemporalBench
+```
+It includes all test tasks and a `forecast_metrics_utils.py` file that documents the
+standard metric computation utilities.
 """
 
 EVALUATION_QUEUE_TEXT = ""

src/leaderboard/load_results.py

@@ -20,6 +20,14 @@ def _is_number(value) -> bool:
     return math.isfinite(float(value))
 
 
+def _is_missing(value) -> bool:
+    if value is None:
+        return True
+    if isinstance(value, float) and math.isnan(value):
+        return True
+    return False
+
+
 def _load_json_records(path: str) -> list[dict]:
     with open(path, "r") as fp:
         data = json.load(fp)
@@ -80,6 +88,8 @@ def validate_records(records: Iterable[dict]) -> None:
         for key, value in record.items():
             if key in SCHEMA.identity_fields:
                 continue
+            if _is_missing(value):
+                continue
             if not _is_number(value):
                 raise ResultsValidationError(
                     f"Record {idx} metric '{key}' must be numeric."
@@ -114,5 +124,86 @@ def build_dataframe(records: list[dict]) -> tuple[pd.DataFrame, list[str]]:
     metric_cols = infer_metric_columns(records)
     column_order = list(SCHEMA.identity_fields) + metric_cols
     df = pd.DataFrame.from_records(records)
+    df = apply_overall_metrics(df)
+    # Include computed columns (e.g., overall_mcq_acc) in display order if present.
+    for col in df.columns:
+        if col in SCHEMA.identity_fields:
+            continue
+        if col not in column_order:
+            column_order.append(col)
     df = df[column_order]
     return df, column_order
+
+
+MCQ_QUESTIONS = {
+    "MIMIC": {"T1": 188, "T2": 141, "T3": 239, "T4": 141},
+    "PSML": {"T1": 200, "T2": 150, "T3": 250, "T4": 150},
+    "CausalChambers": {"T1": 150, "T2": 150, "T3": 250, "T4": 150},
+    "FreshRetailNet": {"T1": 176, "T2": 132, "T3": 176, "T4": 132},
+}
+
+FORECAST_SERIES = {
+    "MIMIC": {"T2": 282, "T4": 282},
+    "PSML": {"T2": 50, "T4": 50},
+    "CausalChambers": {"T2": 50, "T4": 50},
+    "FreshRetailNet": {"T2": 44, "T4": 44},
+}
+
+
+def _weighted_avg(row: pd.Series, columns: list[str], weights: list[int]) -> float | None:
+    total = 0.0
+    total_w = 0.0
+    for col, w in zip(columns, weights):
+        val = row.get(col)
+        if _is_missing(val):
+            continue
+        total += float(val) * w
+        total_w += w
+    if total_w == 0:
+        return None
+    return round(total / total_w, 4)
+
+
+def apply_overall_metrics(df: pd.DataFrame) -> pd.DataFrame:
+    df = df.copy()
+
+    for task in ["T1", "T2", "T3", "T4"]:
+        cols = []
+        weights = []
+        for dataset, task_weights in MCQ_QUESTIONS.items():
+            col = f"{dataset}_{task}_acc"
+            if col in df.columns:
+                cols.append(col)
+                weights.append(task_weights[task])
+        if cols:
+            df[f"{task}_acc"] = df.apply(lambda r: _weighted_avg(r, cols, weights), axis=1)
+
+    overall_cols = []
+    overall_weights = []
+    for dataset, task_weights in MCQ_QUESTIONS.items():
+        for task, weight in task_weights.items():
+            col = f"{dataset}_{task}_acc"
+            if col in df.columns:
+                overall_cols.append(col)
+                overall_weights.append(weight)
+    if overall_cols:
+        df["overall_mcq_acc"] = df.apply(
+            lambda r: _weighted_avg(r, overall_cols, overall_weights), axis=1
+        )
+
+    for task in ["T2", "T4"]:
+        # sMAPE/MAE are defined for non-MIMIC datasets
+        for metric in ["sMAPE", "MAE"]:
+            cols = []
+            weights = []
+            for dataset, task_weights in FORECAST_SERIES.items():
+                if dataset == "MIMIC":
+                    continue
+                col = f"{dataset}_{task}_{metric}"
+                if col in df.columns:
+                    cols.append(col)
+                    weights.append(task_weights[task])
+            if cols:
+                df[f"{task}_{metric}"] = df.apply(lambda r: _weighted_avg(r, cols, weights), axis=1)
+
+    return df

src/leaderboard/schema.py

@@ -6,7 +6,6 @@ from dataclasses import dataclass
 @dataclass(frozen=True)
 class TemporalBenchSchema:
     identity_fields: tuple[str, ...] = (
-        "model_name",
         "agent_name",
         "agent_type",
         "base_model",
@@ -18,8 +17,47 @@ class TemporalBenchSchema:
         "T4_acc",
     )
     optional_metrics: tuple[str, ...] = (
+        "overall_mcq_acc",
+        "T2_sMAPE",
         "T2_MAE",
+        "T2_OW_sMAPE_MIMIC",
+        "T2_OW_RMSSE_MIMIC",
         "T4_sMAPE",
+        "T4_MAE",
+        "T4_OW_sMAPE_MIMIC",
+        "T4_OW_RMSSE_MIMIC",
+        "FreshRetailNet_T1_acc",
+        "FreshRetailNet_T2_acc",
+        "FreshRetailNet_T3_acc",
+        "FreshRetailNet_T4_acc",
+        "PSML_T1_acc",
+        "PSML_T2_acc",
+        "PSML_T3_acc",
+        "PSML_T4_acc",
+        "CausalChambers_T1_acc",
+        "CausalChambers_T2_acc",
+        "CausalChambers_T3_acc",
+        "CausalChambers_T4_acc",
+        "MIMIC_T1_acc",
+        "MIMIC_T2_acc",
+        "MIMIC_T3_acc",
+        "MIMIC_T4_acc",
+        "FreshRetailNet_T2_sMAPE",
+        "FreshRetailNet_T2_MAE",
+        "PSML_T2_sMAPE",
+        "PSML_T2_MAE",
+        "CausalChambers_T2_sMAPE",
+        "CausalChambers_T2_MAE",
+        "MIMIC_T2_OW_sMAPE",
+        "MIMIC_T2_OW_RMSSE",
+        "FreshRetailNet_T4_sMAPE",
+        "FreshRetailNet_T4_MAE",
+        "PSML_T4_sMAPE",
+        "PSML_T4_MAE",
+        "CausalChambers_T4_sMAPE",
+        "CausalChambers_T4_MAE",
+        "MIMIC_T4_OW_sMAPE",
+        "MIMIC_T4_OW_RMSSE",
     )