import json import os import sys from html import escape from pathlib import Path import gradio as gr SPACE_DIR = Path(__file__).resolve().parent if str(SPACE_DIR) not in sys.path: sys.path.insert(0, str(SPACE_DIR)) from src.about import ( CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE, ) from src.display.css_html_js import custom_css from src.utils import ( build_leaderboard_summary_html, build_domain_status_md, dataset_section_title, get_grouped_dfs, group_datasets_by_domain, load_fixed_window_table, load_gift_aggregate_metadata_md, load_gift_aggregate_table, load_forecast_snapshots, make_domain_pie_chart, make_forecast_plot, prepare_ranks_table, prepare_values_table, ) RESULTS_PATH = os.getenv("TSFM_RESULTS_PATH", "space/results" if Path("space/results").exists() else "results") REFRESH_SECONDS = int(os.getenv("TSFM_REFRESH_SECONDS", "300")) DOMAIN_OUTPUT_OFFSET = 14 MODEL_COLUMN_WIDTH = "180px" RANK_MODEL_COLUMN_WIDTH = MODEL_COLUMN_WIDTH RANK_INDEX_COLUMN_WIDTH = MODEL_COLUMN_WIDTH METRIC_COLUMN_WIDTH = "118px" MODEL_COLUMN_NAMES = {"model"} def _fixed_window_outputs(window: str) -> tuple[str, str]: return ( _rank_table_html(load_fixed_window_table(RESULTS_PATH, window, "rank")), _rank_table_html(load_fixed_window_table(RESULTS_PATH, window, "overall")), ) def _model_column_index(value) -> int | None: columns = getattr(value, "columns", None) if columns is None: return None for idx, column in enumerate(columns): if str(column).strip().lower() in MODEL_COLUMN_NAMES: return idx return None def _model_cell_html(value) -> str: text = "" if value is None else str(value) return ( '
' + escape(text) + "
" ) def _rank_display_table(value): model_col_idx = _model_column_index(value) if model_col_idx is None: return value columns = getattr(value, "columns", None) if columns is None: return value table = value.copy() model_col = columns[model_col_idx] table[model_col] = table[model_col].map(_model_cell_html) return table def _dataframe_datatypes(value, *, rank_table: bool = False) -> list[str] | None: columns = getattr(value, "columns", None) if columns is None: return None try: column_count = len(columns) except TypeError: return None if column_count <= 0: return None datatypes = ["str"] * column_count model_col_idx = _model_column_index(value) if rank_table and model_col_idx is not None: datatypes[model_col_idx] = "html" return datatypes def _dataframe_column_widths(value, *, rank_table: bool = False) -> list[str] | None: columns = getattr(value, "columns", None) if columns is None: return None try: column_count = len(columns) except TypeError: return None if column_count <= 0: return None model_width = RANK_MODEL_COLUMN_WIDTH if rank_table else MODEL_COLUMN_WIDTH widths = [METRIC_COLUMN_WIDTH] * column_count model_col_idx = _model_column_index(value) if model_col_idx is None: model_col_idx = 0 widths[model_col_idx] = model_width return widths def _leaderboard_dataframe(value, *, rank_table: bool = False, **kwargs): classes = kwargs.pop("elem_classes", []) or [] if isinstance(classes, str): classes = [classes] if "leaderboard-dataframe" not in classes: classes.append("leaderboard-dataframe") if rank_table and "rank-leaderboard-dataframe" not in classes: classes.append("rank-leaderboard-dataframe") model_col_idx = _model_column_index(value) if model_col_idx is not None: model_col_class = f"leaderboard-model-col-{model_col_idx}" if model_col_class not in classes: classes.append(model_col_class) kwargs["elem_classes"] = classes kwargs["interactive"] = False kwargs["wrap"] = False kwargs["line_breaks"] = False kwargs.setdefault("datatype", _dataframe_datatypes(value, rank_table=rank_table)) kwargs.setdefault("column_widths", _dataframe_column_widths(value, rank_table=rank_table)) return gr.Dataframe(value=value, **kwargs) def _rank_table_html(table) -> str: columns = list(getattr(table, "columns", [])) if not columns: return '
' model_col_idx = _model_column_index(table) if model_col_idx is None: fixed_col_idx = 0 fixed_col_width = RANK_INDEX_COLUMN_WIDTH else: fixed_col_idx = model_col_idx fixed_col_width = RANK_MODEL_COLUMN_WIDTH colgroup = [] for idx in range(len(columns)): width = fixed_col_width if idx == fixed_col_idx else METRIC_COLUMN_WIDTH colgroup.append(f'') header_cells = [] for idx, column in enumerate(columns): class_name = ' class="rank-fixed-column"' if idx == fixed_col_idx else "" header_cells.append(f"{escape(str(column))}") body_rows = [] for _, row in table.iterrows(): cells = [] for idx, column in enumerate(columns): text = "" if row[column] is None else str(row[column]) if idx == fixed_col_idx: cells.append( '
' + escape(text) + "
" ) else: cells.append(f"{escape(text)}") body_rows.append("" + "".join(cells) + "") return ( '
' '' "" + "".join(colgroup) + "" + "".join(header_cells) + "" + "".join(body_rows) + "
" ) def _dataset_tables(grouped: dict, datasets: list[str]) -> tuple[list, list]: values_tables = [ prepare_values_table(grouped["dataset_values"].get(dataset, None)) for dataset in datasets ] rank_tables = [ _rank_table_html(prepare_ranks_table(grouped["dataset_ranks"].get(dataset, None))) for dataset in datasets ] return values_tables, rank_tables def _model_label(root: Path, model_slug: str) -> str: config_path = root / model_slug / "config.json" if not config_path.exists(): return model_slug try: config = json.loads(config_path.read_text()) return str(config.get("model") or model_slug) except Exception: return model_slug def _forecast_choices(root_dir: str, datasets: list[str]) -> dict[str, dict[str, dict]]: dataset_keys = {dataset.split("/")[0] for dataset in datasets} choices: dict[str, dict[str, dict]] = {dataset_key: {} for dataset_key in dataset_keys} root = Path(root_dir) if not root.exists(): return choices for model_dir in sorted(root.iterdir()): if not model_dir.is_dir(): continue model_label = _model_label(root, model_dir.name) for snapshot_dataset, snapshot in load_forecast_snapshots(root_dir, model_dir.name).items(): dataset_key = str(snapshot_dataset).split("/")[0] if dataset_key not in choices: continue choices[dataset_key][model_label] = snapshot return choices def _available_forecast_models(dataset_key: str) -> list[str]: return sorted(FORECAST_CHOICES.get(dataset_key, {}).keys()) def _forecast_plot_for(dataset_key: str, model_label: str): if not model_label: return None return make_forecast_plot(FORECAST_CHOICES.get(dataset_key, {}).get(model_label)) def _make_forecast_plotter(dataset_key: str): def update_forecast_plot(model_label: str): return _forecast_plot_for(dataset_key, model_label) return update_forecast_plot def refresh_leaderboard() -> tuple: grouped = get_grouped_dfs(RESULTS_PATH) datasets = grouped["datasets"] one_day_rank, one_day_metrics = _fixed_window_outputs("1d") seven_day_rank, seven_day_metrics = _fixed_window_outputs("7d") thirty_day_rank, thirty_day_metrics = _fixed_window_outputs("30d") outputs: list = [ build_leaderboard_summary_html(RESULTS_PATH), # [0] prepare_values_table(grouped["overall_values"]), # [1] _rank_table_html(prepare_ranks_table(grouped["overall_ranks"])), # [2] make_domain_pie_chart(RESULTS_PATH), # [3] one_day_rank, # [4] one_day_metrics, # [5] seven_day_rank, # [6] seven_day_metrics, # [7] thirty_day_rank, # [8] thirty_day_metrics, # [9] load_gift_aggregate_metadata_md(RESULTS_PATH), # [10] load_gift_aggregate_table(RESULTS_PATH, "prediction_length"), # [11] load_gift_aggregate_table(RESULTS_PATH, "domain"), # [12] load_gift_aggregate_table(RESULTS_PATH, "frequency"), # [13] ] # [14 .. 14+D-1] per-domain status lines for domain, domain_datasets in DOMAIN_GROUPS.items(): outputs.append(build_domain_status_md(domain, domain_datasets, RESULTS_PATH)) value_tables, rank_tables = _dataset_tables(grouped, datasets) # [14+D .. 14+D+N-1] value tables outputs.extend(value_tables) # [14+D+N .. 14+D+2N-1] rank tables outputs.extend(rank_tables) return tuple(outputs) grouped_initial = get_grouped_dfs(RESULTS_PATH) DATASETS: list[str] = grouped_initial["datasets"] FORECAST_CHOICES = _forecast_choices(RESULTS_PATH, DATASETS) DOMAIN_GROUPS = group_datasets_by_domain(DATASETS, RESULTS_PATH) N_DOMAINS = len(DOMAIN_GROUPS) INITIAL_OUTPUTS = refresh_leaderboard() demo = gr.Blocks(css=custom_css) with demo: gr.HTML(TITLE) gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text") summary_html = gr.HTML(INITIAL_OUTPUTS[0]) dataset_value_dfs: list[gr.Dataframe] = [] dataset_rank_dfs: list[gr.HTML] = [] domain_status_mds: list[gr.Markdown] = [] dataset_idx = 0 with gr.Tabs(elem_classes="tab-buttons"): # ── Overall Tab ────────────────────────────────────────────────────── with gr.TabItem("Overall"): gr.Markdown( "**Latest evaluation snapshot**", elem_classes="markdown-text", ) gr.Markdown( "**Overall summary.** Geometric mean of absolute metric values across the latest dataset snapshot. MSE/CRPS are lower-is-better; RankScore is higher-is-better.", elem_classes="markdown-text", ) overall_values_df = _leaderboard_dataframe( value=INITIAL_OUTPUTS[1], label="Metric values and RankScore", ) gr.Markdown( "**Latest snapshot ranks.** Per-metric ranks derived from the snapshot values above; lower rank is better.", elem_classes="markdown-text", ) overall_ranks_df = gr.HTML( value=INITIAL_OUTPUTS[2], elem_classes="rank-html-output", ) gr.Markdown( "**Evaluated timestamps by domain** · hover for dataset names", elem_classes="markdown-text", ) domain_pie_plot = gr.Plot( value=INITIAL_OUTPUTS[3], label="Domain distribution", ) gr.Markdown( "**Pairwise historical ranking.** Every Ranking table uses all shared releases completed through the current cutoff, balanced equally across datasets. The 24-hour, 7-day, and 30-day windows apply only to their Metrics tables. Official ranks require at least 30 shared releases, 5 shared datasets, 7 days of shared coverage, 3 eligible opponents, and membership in the main comparison component. Absolute metrics remain descriptive and do not determine rank.", elem_classes="markdown-text", ) gr.Markdown( "**Last 24 hours** · cumulative historical ranking; metrics use releases whose target window ended in the trailing 24 hours.", elem_classes="markdown-text", ) gr.Markdown("**Ranking**", elem_classes="markdown-text") last_24h_rank_df = gr.HTML( value=INITIAL_OUTPUTS[4], elem_classes="rank-html-output", ) gr.Markdown("**Metrics**", elem_classes="markdown-text") last_24h_metrics_df = gr.HTML( value=INITIAL_OUTPUTS[5], elem_classes="rank-html-output", ) gr.Markdown( "**Last 7 days** · cumulative historical ranking; metrics use a fixed trailing seven-day window.", elem_classes="markdown-text", ) gr.Markdown("**Ranking**", elem_classes="markdown-text") last_7d_rank_df = gr.HTML( value=INITIAL_OUTPUTS[6], elem_classes="rank-html-output", ) gr.Markdown("**Metrics**", elem_classes="markdown-text") last_7d_metrics_df = gr.HTML( value=INITIAL_OUTPUTS[7], elem_classes="rank-html-output", ) gr.Markdown( "**Last 30 days** · cumulative historical ranking; metrics use a fixed trailing thirty-day window.", elem_classes="markdown-text", ) gr.Markdown("**Ranking**", elem_classes="markdown-text") last_30d_rank_df = gr.HTML( value=INITIAL_OUTPUTS[8], elem_classes="rank-html-output", ) gr.Markdown("**Metrics**", elem_classes="markdown-text") last_30d_metrics_df = gr.HTML( value=INITIAL_OUTPUTS[9], elem_classes="rank-html-output", ) # ── GIFT-style grouped result tables ──────────────────────────────── with gr.TabItem("GIFT-style Aggregates"): aggregate_metadata_md = gr.Markdown( INITIAL_OUTPUTS[10], elem_classes="markdown-text", ) gr.Markdown( "MSE and CRPS are normalized per dataset configuration against Seasonal-Naive (1.0 = baseline). Rank is the mean per-configuration CRPS rank. Lower is better for all three metrics.", elem_classes="markdown-text", ) with gr.Tabs(): with gr.TabItem("Prediction Length"): prediction_length_aggregate_df = _leaderboard_dataframe( value=INITIAL_OUTPUTS[11], label="Results on TSFM_Bench aggregated by Prediction Length", ) with gr.TabItem("Domain"): domain_aggregate_df = _leaderboard_dataframe( value=INITIAL_OUTPUTS[12], label="Results on TSFM_Bench aggregated by Domain", ) with gr.TabItem("Frequency"): frequency_aggregate_df = _leaderboard_dataframe( value=INITIAL_OUTPUTS[13], label="Results on TSFM_Bench aggregated by Frequency", ) # ── Domain Tabs ─────────────────────────────────────────────────────── domain_idx = 0 for domain, domain_datasets in DOMAIN_GROUPS.items(): domain_label = f"{domain} ({len(domain_datasets)})" with gr.TabItem(domain_label): # Per-domain status line (data source, last eval, refresh intervals) domain_status_mds.append( gr.Markdown( INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + domain_idx], elem_classes="markdown-text", ) ) domain_idx += 1 for dataset in domain_datasets: gr.Markdown( dataset_section_title(dataset, RESULTS_PATH), elem_classes="dataset-heading", ) dataset_value_dfs.append( _leaderboard_dataframe( value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + dataset_idx], show_label=False, ) ) gr.Markdown( dataset_section_title(dataset, RESULTS_PATH, ranks=True), elem_classes="dataset-heading", ) dataset_rank_dfs.append( gr.HTML( value=INITIAL_OUTPUTS[DOMAIN_OUTPUT_OFFSET + N_DOMAINS + len(DATASETS) + dataset_idx], elem_classes="rank-html-output", ) ) dataset_key = dataset.split("/")[0] forecast_model_choices = _available_forecast_models(dataset_key) default_forecast_model = forecast_model_choices[0] if forecast_model_choices else None forecast_model_dropdown = gr.Dropdown( choices=forecast_model_choices, value=default_forecast_model, label="Model", interactive=bool(forecast_model_choices), ) forecast_plot = gr.Plot( value=_forecast_plot_for(dataset_key, default_forecast_model), label="Forecast snapshot", ) forecast_model_dropdown.change( fn=_make_forecast_plotter(dataset_key), inputs=[forecast_model_dropdown], outputs=[forecast_plot], ) dataset_idx += 1 with gr.Accordion("About", open=False): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") with gr.Accordion("Citation", open=False): gr.Textbox( value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=12, show_copy_button=True, ) refresh_outputs = [ summary_html, overall_values_df, overall_ranks_df, domain_pie_plot, last_24h_rank_df, last_24h_metrics_df, last_7d_rank_df, last_7d_metrics_df, last_30d_rank_df, last_30d_metrics_df, aggregate_metadata_md, prediction_length_aggregate_df, domain_aggregate_df, frequency_aggregate_df, *domain_status_mds, *dataset_value_dfs, *dataset_rank_dfs, ] demo.load(refresh_leaderboard, inputs=[], outputs=refresh_outputs) gr.Timer(value=REFRESH_SECONDS).tick(refresh_leaderboard, inputs=[], outputs=refresh_outputs) if __name__ == "__main__": demo.queue(default_concurrency_limit=20).launch()