'
@@ -367,18 +499,19 @@ def _leaderboard_html(data, columns, score_columns, overall_column):
header_cells = []
for index, column in enumerate(leaderboard.columns):
sort_type = _leaderboard_sort_type(column)
+ cell_class = _leaderboard_cell_class(column)
header_cells.append(
- f'
"
)
body_rows = []
for _, row in leaderboard.iterrows():
cells = []
for column in leaderboard.columns:
- css = "rank" if column == "Rank" else "metric-score"
+ css = _leaderboard_cell_class(column)
sort_value = escape(_leaderboard_sort_value(column, row[column]), quote=True)
if column == "Model":
- css = "model-cell"
cells.append(
f'
"
@@ -402,130 +535,17 @@ def _leaderboard_html(data, columns, score_columns, overall_column):
"""
-def render_leaderboard(
- data,
- columns,
- note=None,
- score_columns=None,
- overall_column=None,
-):
- score_columns = list(score_columns or _infer_score_columns(columns))
- overall_column = overall_column or _default_overall_column(score_columns)
- platform_choices = _filter_choices(data, "Platform")
- owner_choices = _filter_choices(data, "Endpoint Owner")
- optimized_choices = _filter_choices(data, "Optimized")
-
- if note:
- gr.Markdown(note)
-
- filter_inputs = []
- with gr.Row(elem_classes="leaderboard-controls"):
- search = gr.Textbox(
- label="Search models",
- placeholder="Search by model or provider",
- scale=3,
- )
- filter_inputs.append(search)
- platform = None
- owner = None
- optimized = None
- if platform_choices:
- platform = gr.Dropdown(
- choices=platform_choices,
- value=[],
- label="Providers",
- multiselect=True,
- scale=1,
- )
- filter_inputs.append(platform)
- if owner_choices:
- owner = gr.Dropdown(
- choices=owner_choices,
- value=[],
- label="Endpoint owners",
- multiselect=True,
- scale=1,
- )
- filter_inputs.append(owner)
- if optimized_choices:
- optimized = gr.Dropdown(
- choices=optimized_choices,
- value=[],
- label="Optimized",
- multiselect=True,
- scale=1,
- )
- filter_inputs.append(optimized)
-
- ranking = gr.HTML(
- _leaderboard_html(data, columns, score_columns, overall_column),
- elem_classes="ranking-table-host",
- )
-
- def update_ranking(
- search_term,
- platform_value=None,
- owner_value=None,
- optimized_value=None,
- ):
- filtered_data = _filter_leaderboard(
- data,
- search_term,
- platform_value or [],
- owner_value or [],
- optimized_value or [],
- )
- return _leaderboard_html(
- filtered_data, columns, score_columns, overall_column
- )
-
- # Wire only the filters that actually exist for this table.
- change_inputs = [search]
- if platform is not None:
- change_inputs.append(platform)
- if owner is not None:
- change_inputs.append(owner)
- if optimized is not None:
- change_inputs.append(optimized)
-
- for component in filter_inputs:
- component.change(
- update_ranking,
- inputs=change_inputs,
- outputs=ranking,
- )
-
-
-def _infer_score_columns(columns):
- return [column for column in columns if column.startswith("OneIG (")]
-
-
-def _default_overall_column(score_columns):
- if len(score_columns) == 1:
- return score_columns[0]
- return "OneIG Overall Score"
-
-
def _filter_choices(data, column):
- if column not in data.columns:
+ if data is None or column not in data.columns:
return []
return sorted(data[column].dropna().astype(str).unique().tolist())
-def _filter_leaderboard(data, search_term, platform, owner, optimized):
+def _filter_leaderboard(data, platform, owner, optimized, models=None):
filtered = data.copy()
- if search_term:
- search_columns = [
- column
- for column in ["Model", "Platform", "Endpoint Owner"]
- if column in filtered.columns
- ]
- matches = pd.Series(False, index=filtered.index)
- for column in search_columns:
- matches |= filtered[column].astype(str).str.contains(
- search_term, case=False, na=False
- )
- filtered = filtered[matches]
+ if models:
+ if "Model" in filtered.columns:
+ filtered = filtered[filtered["Model"].astype(str).isin(models)]
for column, values in [
("Platform", platform),
@@ -537,64 +557,22 @@ def _filter_leaderboard(data, search_term, platform, owner, optimized):
return filtered
-def _leaderboard_dataframe(data, columns, score_columns, overall_column):
- # Honor the caller-provided column list so extra metrics (e.g. Elo) are not
- # dropped just because they are not part of the ranking score_columns.
- skip_columns = {"URL", "Rank"}
- preferred_prefix = [
- column
- for column in ["Model", "Platform", "Endpoint Owner", "Optimized"]
- if column in data.columns
- ]
- preferred_suffix = [
- column
- for column in [
- "Median Generation Time (s)",
- "Min Generation Time (s)",
- "Price / Image (USD)",
- "Evaluation Date (UTC)",
- "Date",
- ]
- if column in data.columns
- ]
- # Keep overall_column visible when the caller includes it (e.g. Datapoint Elo).
- # Synthetic aggregates like OneIG Overall Score are simply omitted from `columns`.
- middle = [
- column
- for column in columns
- if column in data.columns
- and column not in skip_columns
- and column not in preferred_prefix
- and column not in preferred_suffix
- ]
-
- ordered_columns = []
- seen = set()
- for column in [*preferred_prefix, *middle, *preferred_suffix]:
- if column not in seen:
- seen.add(column)
- ordered_columns.append(column)
-
- leaderboard = data[ordered_columns].copy()
-
- # Rank by overall when available, even if that column is not displayed.
- if overall_column and overall_column in data.columns:
- leaderboard = (
- leaderboard.assign(_sort_key=data[overall_column])
- .sort_values("_sort_key", ascending=False, na_position="last")
- .drop(columns=["_sort_key"])
- .reset_index(drop=True)
- )
- else:
- leaderboard = leaderboard.reset_index(drop=True)
-
- leaderboard.insert(0, "Rank", leaderboard.index + 1)
- return leaderboard.rename(columns=_display_label)
+def _leaderboard_dataframe(data, columns):
+ ordered = []
+ if "Rank" in data.columns:
+ ordered.append("Rank")
+ for column in columns:
+ if (
+ column in data.columns
+ and column not in {"URL", "Rank"}
+ and column not in ordered
+ ):
+ ordered.append(column)
+ return data[ordered].rename(columns=_display_label)
def _display_label(column):
labels = {
- "_overall_score": "Overall score",
"OneIG Overall Score": "Overall",
"OneIG (Anime Alignment)": "Anime",
"OneIG (Human Alignment)": "Human",
@@ -602,10 +580,18 @@ def _display_label(column):
"OneIG Anime Elo": "Anime Elo (Datapoint)",
"OneIG Human Elo": "Human Elo (Datapoint)",
"OneIG Object Elo": "Object Elo (Datapoint)",
- "P-Judge Overall": "P-Judge",
+ "P-Judge Overall": "P-Judger (Pruna)",
"Datapoint Elo": "Datapoint Elo",
"Rapidata Elo": "Rapidata Elo",
- "Benchmark.ai Elo": "Benchmark.ai Elo",
+ "Artificial Analysis Elo": "Artificial Analysis Elo",
+ "Arena Elo": "Overall Elo",
+ "Arena Branding / Commercial Elo": "Branding / Commercial",
+ "Arena 3D Imaging Elo": "3D Imaging",
+ "Arena Cartoon / Anime Elo": "Cartoon / Anime",
+ "Arena Photorealistic Elo": "Photorealistic",
+ "Arena Art Elo": "Art",
+ "Arena Portraits Elo": "Portraits",
+ "Arena Text Rendering Elo": "Text Rendering",
"Raw Win Rate": "Raw win rate",
"Median Generation Time (s)": "Median generation time",
"Min Generation Time (s)": "Min generation time",
@@ -616,158 +602,459 @@ def _display_label(column):
return labels.get(column, column)
-def _text_value(value):
- return "β" if pd.isna(value) or value is None else escape(str(value))
+def _format_price(value):
+ return "-" if pd.isna(value) or value is None else f"${float(value):.3f}"
-def _format_score(value):
- return "β" if pd.isna(value) or value is None else f"{float(value):.3f}"
+def _pareto_frontier_mask(x_values, scores):
+ """True for non-dominated points when maximizing score and minimizing x."""
+ n = len(x_values)
+ mask = [True] * n
+ for i in range(n):
+ for j in range(n):
+ if i == j:
+ continue
+ better_or_equal = x_values[j] <= x_values[i] and scores[j] >= scores[i]
+ strictly_better = x_values[j] < x_values[i] or scores[j] > scores[i]
+ if better_or_equal and strictly_better:
+ mask[i] = False
+ break
+ return mask
-def _format_price(value):
- return "β" if pd.isna(value) or value is None else f"${float(value):.3f}"
+def _pareto_unavailable_html(message):
+ return (
+ "
"
+ )
-def render_benchmark_detail(benchmark):
- gr.Markdown(
- f"""
- # {benchmark["title"]}
+def _pareto_note_update(message):
+ if message:
+ return gr.update(
+ value=_pareto_unavailable_html(message),
+ visible=True,
+ )
+ return gr.update(value="", visible=False)
- {benchmark["intro"]}
- """
- )
- with gr.Tabs(elem_classes="subtabs", selected=0) as detail_tabs:
- with gr.TabItem("Leaderboard"):
- render_leaderboard(
- benchmark["data"],
- benchmark["columns"],
- note=benchmark.get("note"),
- score_columns=benchmark.get("score_columns"),
- overall_column=benchmark.get("overall_column"),
- )
- with gr.TabItem("Graphs"):
- render_benchmark_graphs(benchmark)
- with gr.TabItem("Compare samples"):
- render_compare_samples(benchmark)
- return detail_tabs
+def _pareto_plot_update(fig):
+ if fig is not None:
+ return gr.update(value=fig, visible=True)
+ return gr.update(value=None, visible=False)
-def render_compare_samples(benchmark):
- samples = benchmark.get("samples")
- if not samples:
- gr.Markdown(
- """
- Sample comparison is not available for this benchmark yet.
- When generations are linked, you will be able to pick models and browse
- side-by-side outputs for the same prompts.
- """
- )
- return
+def _skip_all(count):
+ return tuple(gr.skip() for _ in range(count))
- models = samples["models"]
- default_models = models[: min(2, len(models))]
- gr.Markdown(
- f"""
-
- """
- )
- with gr.Row(elem_classes="leaderboard-controls"):
- model_picker = gr.Dropdown(
- choices=models,
- value=default_models,
- multiselect=True,
- max_choices=MAX_COMPARE_MODELS,
- label="Models",
- info=f"Select 1β{MAX_COMPARE_MODELS} models to compare",
- scale=3,
- )
- prompt_count = gr.Slider(
- minimum=1,
- maximum=MAX_COMPARE_PROMPTS,
- value=DEFAULT_COMPARE_PROMPTS,
- step=1,
- label="Prompts to show",
- scale=1,
- )
- shuffle_button = gr.Button("Shuffle prompts", scale=1)
-
- gallery = gr.HTML(
- value=_build_compare_samples_html(
- samples,
- default_models,
- DEFAULT_COMPARE_PROMPTS,
- seed=0,
- )
- )
- seed_state = gr.State(0)
-
- def update_gallery(selected_models, num_prompts, seed):
- return _build_compare_samples_html(
- samples,
- selected_models,
- int(num_prompts),
- seed=int(seed or 0),
- )
+def _pareto_skip_updates():
+ return _skip_all(_PARETO_SLOT_COUNT)
- def shuffle_gallery(selected_models, num_prompts, seed):
- next_seed = int(seed or 0) + 1
- return next_seed, _build_compare_samples_html(
- samples,
- selected_models,
- int(num_prompts),
- seed=next_seed,
- )
- model_picker.change(
- update_gallery,
- inputs=[model_picker, prompt_count, seed_state],
- outputs=gallery,
+def _selection_key(dataset_id, metric_id, models):
+ return (
+ dataset_id,
+ tuple(_normalize_metric_ids(metric_id)),
+ tuple(models or ()),
)
- prompt_count.change(
- update_gallery,
- inputs=[model_picker, prompt_count, seed_state],
- outputs=gallery,
+
+
+def _applied_key(view_state):
+ view_state = view_state or {}
+ return _selection_key(
+ view_state.get("dataset_id"),
+ view_state.get("metric_id"),
+ view_state.get("models"),
)
- shuffle_button.click(
- shuffle_gallery,
- inputs=[model_picker, prompt_count, seed_state],
- outputs=[seed_state, gallery],
+
+
+def _build_pareto_figure(
+ data,
+ score_column,
+ x_column,
+ x_title,
+ x_hover_prefix="",
+ x_hover_suffix="",
+ x_axis_type="linear",
+):
+ scatter = (
+ data[["Model", score_column, x_column]]
+ .dropna()
+ .copy()
+ .reset_index(drop=True)
)
+ if scatter.empty:
+ return None
+ x_values = scatter[x_column].astype(float).tolist()
+ scores = scatter[score_column].astype(float).tolist()
+ on_frontier = _pareto_frontier_mask(x_values, scores)
-def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
- selected_models = [
- model
- for model in (selected_models or [])
- if model in samples["images"]
- ][:MAX_COMPARE_MODELS]
+ dominated = scatter.loc[[not flag for flag in on_frontier]].copy()
+ frontier = scatter.loc[on_frontier].sort_values(x_column).copy()
+ if not dominated.empty:
+ dominated["Model"] = dominated["Model"].map(display_model_name)
+ if not frontier.empty:
+ frontier["Model"] = frontier["Model"].map(display_model_name)
+ hover = (
+ "
"
+ f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"
+ "
"
+ fig = go.Figure()
+ if not dominated.empty:
+ fig.add_trace(
+ go.Scatter(
+ x=dominated[x_column],
+ y=dominated[score_column],
+ mode="markers",
+ name="Below frontier",
+ text=dominated["Model"],
+ hovertemplate=hover,
+ marker={
+ "size": 9,
+ "color": "#d8b4fe",
+ "opacity": 0.8,
+ "line": {"width": 0},
+ },
+ )
)
-
- shared_prompt_ids = None
- for model in selected_models:
- model_prompt_ids = set(samples["images"][model])
- shared_prompt_ids = (
- model_prompt_ids
- if shared_prompt_ids is None
- else shared_prompt_ids & model_prompt_ids
+ if not frontier.empty:
+ fig.add_trace(
+ go.Scatter(
+ x=frontier[x_column],
+ y=frontier[score_column],
+ mode="lines+markers",
+ name="On frontier",
+ text=frontier["Model"],
+ hovertemplate=hover,
+ line={"color": "#69a45c", "width": 2.5},
+ marker={
+ "size": 12,
+ "color": "#69a45c",
+ "line": {"width": 1.5, "color": "#86c077"},
+ },
+ )
)
- shared_prompt_ids = sorted(shared_prompt_ids or [])
- if not shared_prompt_ids:
- return (
- '
'
+ score_label = _display_label(score_column)
+ fig.update_layout(
+ title=None,
+ xaxis_title=x_title,
+ yaxis_title=score_label,
+ autosize=True,
+ height=420,
+ margin={"l": 56, "r": 28, "t": 28, "b": 80},
+ legend={
+ "orientation": "h",
+ "yanchor": "top",
+ "y": -0.24,
+ "xanchor": "center",
+ "x": 0.5,
+ "bgcolor": "rgba(0,0,0,0)",
+ "font": {"color": "#d4d4d4", "size": 12},
+ },
+ plot_bgcolor="#1d1429",
+ paper_bgcolor="#171021",
+ font={"color": "#d4d4d4", "size": 13},
+ )
+ axis_font = {"color": "#fafafa", "size": 13}
+ tick_font = {"color": "#a3a3a3", "size": 12}
+ x_axis_ticks = {}
+ if x_axis_type == "log":
+ positive_x = scatter.loc[scatter[x_column] > 0, x_column].astype(float)
+ if not positive_x.empty:
+ minimum = positive_x.min()
+ maximum = positive_x.max()
+ tick_values = [
+ factor * (10**exponent)
+ for exponent in range(
+ floor(log10(minimum)),
+ ceil(log10(maximum)) + 1,
+ )
+ for factor in (1, 2, 5)
+ if minimum * 0.8 <= factor * (10**exponent) <= maximum * 1.2
+ ]
+ x_axis_ticks = {
+ "tickmode": "array",
+ "tickvals": tick_values,
+ "ticktext": [f"{value:g}" for value in tick_values],
+ }
+ fig.update_xaxes(
+ type=x_axis_type,
+ **x_axis_ticks,
+ showgrid=True,
+ gridcolor="rgba(74, 57, 98, 0.55)",
+ zeroline=False,
+ title_font=axis_font,
+ tickfont=tick_font,
+ color="#d4d4d4",
+ )
+ fig.update_yaxes(
+ showgrid=True,
+ gridcolor="rgba(74, 57, 98, 0.55)",
+ zeroline=False,
+ title_font=axis_font,
+ tickfont=tick_font,
+ color="#d4d4d4",
+ )
+ return fig
+
+
+def _is_log_scale(scale):
+ return scale == "Logarithmic"
+
+
+def _pareto_axis_type(scale):
+ return "log" if _is_log_scale(scale) else "linear"
+
+
+def _pareto_scale_radio(*extra_classes):
+ return gr.Radio(
+ choices=_PARETO_SCALE_CHOICES,
+ value=_PARETO_SCALE_DEFAULT,
+ show_label=False,
+ container=False,
+ elem_classes=["pareto-scale-toggle", *extra_classes],
+ )
+
+
+def _pareto_plot_heading(title):
+ with gr.Row(equal_height=False, elem_classes="pareto-heading-row"):
+ gr.Markdown(f"#### {title}", elem_classes="pareto-subhead")
+ with gr.Column(min_width=140, elem_classes="pareto-scale-control"):
+ return _pareto_scale_radio()
+
+
+def _default_pareto_scales():
+ return [_PARETO_SCALE_DEFAULT] * MAX_PARETO_METRICS
+
+
+def _normalize_pareto_scales(scales):
+ values = list(scales or [])
+ if len(values) < MAX_PARETO_METRICS:
+ values.extend(
+ [_PARETO_SCALE_DEFAULT] * (MAX_PARETO_METRICS - len(values))
+ )
+ return values[:MAX_PARETO_METRICS]
+
+
+def _uniform_pareto_scales(scale):
+ return [scale] * MAX_PARETO_METRICS
+
+
+def _pareto_master_scale_update(price_scales, time_scales):
+ values = list(price_scales) + list(time_scales)
+ if values and all(value == values[0] for value in values):
+ return gr.update(value=values[0])
+ return gr.update(value=None)
+
+
+def _pareto_axis(data, score_column, x_column, x_title, missing_message, empty_message, **hover):
+ if x_column not in data.columns:
+ return None, missing_message
+ fig = _build_pareto_figure(
+ data,
+ score_column,
+ x_column=x_column,
+ x_title=x_title,
+ **hover,
+ )
+ if fig is None:
+ return None, empty_message
+ return fig, None
+
+
+def _pareto_pair(
+ data,
+ score_column,
+ latency_scale=_PARETO_SCALE_DEFAULT,
+ price_scale=_PARETO_SCALE_DEFAULT,
+):
+ score_missing = "No score data is available for this metric."
+ if data is None or not score_column or score_column not in data.columns:
+ return None, score_missing, None, score_missing
+
+ price_fig, price_message = _pareto_axis(
+ data,
+ score_column,
+ _PARETO_PRICE_COLUMN,
+ "Price per image (USD)",
+ "Price per image isn't available for this dataset.",
+ "No models have both a score and a price for this metric.",
+ x_hover_prefix="$",
+ x_axis_type=_pareto_axis_type(price_scale),
+ )
+ time_fig, time_message = _pareto_axis(
+ data,
+ score_column,
+ _PARETO_TIME_COLUMN,
+ "Min generation time (s)",
+ "Min generation time isn't available for this dataset.",
+ "No models have both a score and a min generation time for this metric.",
+ x_hover_suffix="s",
+ x_axis_type=_pareto_axis_type(latency_scale),
+ )
+ return price_fig, price_message, time_fig, time_message
+
+
+def _pareto_dataset_message(data):
+ has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
+ has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
+ if has_price or has_time:
+ return None
+ return (
+ "Price per image and min generation time aren't available for "
+ "this dataset, so these plots can't be drawn."
+ )
+
+
+def _pareto_slot_note(price_fig, price_message, time_fig, time_message, data):
+ has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
+ has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
+ notes = []
+ if has_price and not has_time:
+ notes.append(
+ "Min generation time isn't available for this dataset, so only "
+ "price vs score is shown."
+ )
+ elif has_time and not has_price:
+ notes.append(
+ "Price per image isn't available for this dataset, so only min "
+ "generation time vs score is shown."
+ )
+ if price_fig is None and has_price:
+ notes.append(price_message)
+ if time_fig is None and has_time:
+ notes.append(time_message)
+ if len(notes) == 2 and notes[0] == notes[1]:
+ notes = notes[:1]
+ return " ".join(notes)
+
+
+def _pareto_slot_updates(
+ data,
+ score_columns,
+ price_scales=None,
+ time_scales=None,
+):
+ """Updates for a fixed bank of Gradio Plot slots (visible/hidden)."""
+ score_columns = [column for column in (score_columns or []) if column]
+ price_scales = _normalize_pareto_scales(price_scales)
+ time_scales = _normalize_pareto_scales(time_scales)
+ has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
+ has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
+ dataset_note = _pareto_dataset_message(data)
+ updates = [_pareto_note_update(dataset_note)]
+ hide_all_slots = not has_price and not has_time
+ hidden_slot = (
+ gr.update(visible=False),
+ "",
+ _pareto_note_update(""),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ _pareto_plot_update(None),
+ gr.update(visible=False),
+ _pareto_plot_update(None),
+ )
+ for index in range(MAX_PARETO_METRICS):
+ if hide_all_slots or index >= len(score_columns):
+ updates.extend(hidden_slot)
+ continue
+ score_column = score_columns[index]
+ price_fig, price_message, time_fig, time_message = _pareto_pair(
+ data,
+ score_column,
+ latency_scale=time_scales[index],
+ price_scale=price_scales[index],
+ )
+ show_price = price_fig is not None
+ show_time = time_fig is not None
+ updates.extend(
+ [
+ gr.update(visible=True),
+ f"#### {_display_label(score_column)}",
+ _pareto_note_update(
+ _pareto_slot_note(
+ price_fig,
+ price_message,
+ time_fig,
+ time_message,
+ data,
+ )
+ ),
+ gr.update(visible=show_price or show_time),
+ gr.update(visible=show_price),
+ _pareto_plot_update(price_fig),
+ gr.update(visible=show_time),
+ _pareto_plot_update(time_fig),
+ ]
+ )
+ return updates
+
+
+def _pareto_all_scale_updates(data, score_columns, scale):
+ """Apply one scale to every Pareto plot and radio."""
+ score_columns = [column for column in (score_columns or []) if column]
+ price_updates = []
+ time_updates = []
+ for index in range(MAX_PARETO_METRICS):
+ if index >= len(score_columns):
+ price_updates.append(gr.skip())
+ time_updates.append(gr.skip())
+ continue
+ price_fig, _, time_fig, _ = _pareto_pair(
+ data,
+ score_columns[index],
+ latency_scale=scale,
+ price_scale=scale,
+ )
+ price_updates.append(_pareto_plot_update(price_fig))
+ time_updates.append(_pareto_plot_update(time_fig))
+ radio_updates = [
+ gr.update(value=scale) for _ in range(MAX_PARETO_METRICS * 2)
+ ]
+ return price_updates + time_updates + radio_updates
+
+
+def _samples_html(samples, selected_models, num_prompts, seed=0):
+ if not samples:
+ return _pareto_unavailable_html(
+ "Samples aren't available for this dataset."
+ )
+ images = samples.get("images", {})
+ models = [model for model in (selected_models or []) if model in images]
+ if not models:
+ models = (samples.get("models") or [])[:2]
+ return _build_compare_samples_html(samples, models, num_prompts, seed)
+
+
+def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
+ selected_models = list(selected_models or [])[:MAX_COMPARE_MODELS]
+
+ if not selected_models:
+ return (
+ '
'
+ "Select at least one model to compare samples."
+ "
"
+ )
+
+ shared_prompt_ids = None
+ for model in selected_models:
+ model_prompt_ids = set(samples["images"][model])
+ shared_prompt_ids = (
+ model_prompt_ids
+ if shared_prompt_ids is None
+ else shared_prompt_ids & model_prompt_ids
+ )
+
+ shared_prompt_ids = sorted(shared_prompt_ids or [])
+ if not shared_prompt_ids:
+ return (
+ '
'
"No shared prompts found for the selected models."
"
"
)
@@ -787,9 +1074,9 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
cells.append(
f"""
-
{escape(model)}
+
{escape(display_model_name(model))}
- 
+
"""
@@ -812,312 +1099,1175 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
return "\n".join(blocks)
-def render_benchmarks(benchmarks):
- """Catalogue cards + detail pages; back button returns to the catalogue."""
- open_buttons = []
- detail_entries = []
+def _leaderboard_intro_markdown(note):
+ parts = [
+ "Models are ranked by the selected metric, with price and generation "
+ "time in the same table."
+ ]
+ extra = (note or "").strip()
+ if extra:
+ parts.append(extra)
+ return "
" + " ".join(parts) + "
"
- with gr.Column(visible=True, elem_classes="benchmark-catalogue") as catalogue:
- gr.Markdown(
- """
- # Benchmarks
- Choose a prompt suite. Each one has a **Leaderboard** table, **Graphs**,
- and **Compare samples**.
- """
+def _filter_row(datasets, metrics, default_dataset_id, default_metric_id=None):
+ metric_id = _coerce_metric(
+ datasets, metrics, default_dataset_id, default_metric_id
+ )
+ with gr.Row(elem_classes="view-filters"):
+ dataset_dd = gr.Dropdown(
+ choices=_dataset_choices(datasets),
+ value=default_dataset_id,
+ label="Dataset",
+ type="value",
+ filterable=False,
+ scale=2,
+ min_width=160,
)
- card_rows = [benchmarks[i : i + 2] for i in range(0, len(benchmarks), 2)]
- for row in card_rows:
- with gr.Row(equal_height=True, elem_classes="benchmark-catalogue-row"):
- for benchmark in row:
- with gr.Column(scale=1, min_width=280):
- with gr.Group(elem_classes="benchmark-card"):
- gr.Markdown(
- f"""
- ## {benchmark.get("emoji", "π")} {benchmark["title"]}
-
- {benchmark["card_description"]}
- """
- )
- open_buttons.append(
- (
- benchmark["id"],
- gr.Button("View benchmark β", variant="primary"),
- )
- )
+ metric_dd = gr.Dropdown(
+ choices=_metric_dropdown_choices(datasets, metrics, default_dataset_id),
+ value=_metric_dropdown_value(metric_id),
+ label="Metric",
+ type="value",
+ multiselect=True,
+ allow_custom_value=False,
+ filterable=True,
+ scale=2,
+ min_width=180,
+ elem_classes="filter-chips",
+ )
+ models_dd = gr.Dropdown(
+ choices=_model_choices(datasets, default_dataset_id),
+ value=[],
+ multiselect=True,
+ label="Models",
+ type="value",
+ allow_custom_value=False,
+ filterable=True,
+ scale=2,
+ min_width=180,
+ elem_classes="filter-chips",
+ )
+ return dataset_dd, metric_dd, models_dd
- for benchmark in benchmarks:
- with gr.Column(visible=False) as detail:
- back_button = gr.Button("β All benchmarks", size="sm")
- render_benchmark_detail(benchmark)
- detail_entries.append((benchmark["id"], detail, back_button))
- nav_outputs = [catalogue, *[detail for _, detail, _ in detail_entries]]
+def render_image_workspace(datasets, metrics, default_dataset_id, default_metric_id):
+ default_metric_id = _coerce_metric(
+ datasets, metrics, default_dataset_id, default_metric_id
+ )
+ initial = resolve_view(datasets, metrics, default_dataset_id, default_metric_id)
+ initial_data = initial["data"]
+ initial_columns = initial["columns"]
+ initial_samples = initial.get("samples")
+ with gr.Column(elem_classes="workspace-shell"):
+ with gr.Column(elem_classes="workspace-filters") as filters_host:
+ gr.Markdown(
+ "
"
+ "These filters apply to Leaderboards, Pareto plots, and Samples. "
+ "On Samples, only datasets we have generations for are listed. "
+ "On Pareto plots, only datasets with price or generation time "
+ "are listed. Search in Models, or leave it empty to include "
+ "every model."
+ "
",
+ elem_classes="filter-help-host",
+ )
+ dataset_dd, metric_dd, models_dd = _filter_row(
+ datasets, metrics, default_dataset_id, None
+ )
+ with gr.Tabs(elem_classes="main-tabs") as main_tabs:
+ with gr.TabItem("Leaderboards", id=TAB_LEADERBOARDS) as lb_tab:
+ lb_note = gr.Markdown(
+ _leaderboard_intro_markdown(initial.get("note")),
+ elem_classes="view-help-host",
+ )
+ platform_choices = _filter_choices(initial_data, "Platform")
+ owner_choices = _filter_choices(initial_data, "Endpoint Owner")
+ optimized_choices = _filter_choices(initial_data, "Optimized")
+ with gr.Row(
+ elem_classes="leaderboard-controls",
+ visible=bool(
+ platform_choices or owner_choices or optimized_choices
+ ),
+ ) as lb_controls:
+ platform = gr.Dropdown(
+ choices=platform_choices,
+ value=[],
+ label="Providers",
+ multiselect=True,
+ allow_custom_value=False,
+ filterable=False,
+ scale=1,
+ visible=bool(platform_choices),
+ )
+ owner = gr.Dropdown(
+ choices=owner_choices,
+ value=[],
+ label="Endpoint owners",
+ multiselect=True,
+ allow_custom_value=False,
+ filterable=False,
+ scale=1,
+ visible=bool(owner_choices),
+ )
+ optimized = gr.Dropdown(
+ choices=optimized_choices,
+ value=[],
+ label="Optimized",
+ multiselect=True,
+ allow_custom_value=False,
+ filterable=False,
+ scale=1,
+ visible=bool(optimized_choices),
+ )
+ ranking = gr.HTML(
+ _leaderboard_html(
+ _assign_leaderboard_ranks(
+ initial_data, initial.get("score_column")
+ ),
+ initial_columns,
+ ),
+ padding=False,
+ elem_classes="ranking-table-host",
+ )
+
+ with gr.TabItem(
+ "Pareto Plots",
+ id=TAB_PARETO,
+ interactive=_dataset_has_pareto(datasets, default_dataset_id),
+ ) as pp_tab:
+ gr.Markdown(
+ "
"
+ "Score against price and generation time. Green points are on "
+ "the frontier; lavender points sit below it. Hover a point to "
+ "see which model it is."
+ "
"
+ "
"
+ "You can switch between linear and logarithmic scale for all "
+ "the plots, or individually for each plots."
+ "
",
+ elem_classes="view-help-host",
+ )
+ with gr.Row(
+ equal_height=False,
+ elem_classes="pareto-scale-all-row",
+ ):
+ gr.HTML(
+ "
All plots",
+ padding=False,
+ )
+ pareto_all_scale = _pareto_scale_radio(
+ "pareto-scale-toggle-all",
+ )
+ pareto_dataset_note = gr.HTML(
+ "",
+ padding=False,
+ visible=False,
+ elem_classes="pareto-note",
+ )
+ pareto_slots = []
+ for slot_index in range(MAX_PARETO_METRICS):
+ with gr.Column(
+ visible=False,
+ elem_classes="pareto-metric-block",
+ ) as slot_group:
+ slot_title = gr.Markdown(
+ "",
+ elem_classes="pareto-metric-title",
+ )
+ with gr.Row(
+ equal_height=True,
+ elem_classes="pareto-layout",
+ ) as slot_layout:
+ with gr.Column(
+ scale=1,
+ min_width=320,
+ elem_classes="pareto-col",
+ ) as slot_price_col:
+ slot_price_scale = _pareto_plot_heading(
+ "Price vs score"
+ )
+ slot_price = gr.Plot(
+ value=None,
+ show_label=False,
+ elem_classes="pareto-plot",
+ )
+ with gr.Column(
+ scale=1,
+ min_width=320,
+ elem_classes="pareto-col",
+ ) as slot_time_col:
+ slot_time_scale = _pareto_plot_heading(
+ "Min generation time vs score"
+ )
+ slot_time = gr.Plot(
+ value=None,
+ show_label=False,
+ elem_classes="pareto-plot",
+ )
+ slot_note = gr.HTML(
+ "",
+ padding=False,
+ visible=False,
+ elem_classes="pareto-note",
+ )
+ pareto_slots.append(
+ (
+ slot_group,
+ slot_title,
+ slot_note,
+ slot_layout,
+ slot_price_col,
+ slot_price,
+ slot_price_scale,
+ slot_time_col,
+ slot_time,
+ slot_time_scale,
+ )
+ )
- def show_catalogue(_evt=None):
+ with gr.TabItem(
+ "Samples",
+ id=TAB_SAMPLES,
+ interactive=_dataset_has_samples(datasets, default_dataset_id),
+ ) as sm_tab:
+ with gr.Column(visible=bool(initial_samples)) as samples_panel:
+ gr.Markdown(
+ f"
"
+ f"The same prompts, side by side. Select up to "
+ f"{MAX_COMPARE_MODELS} models above, or leave "
+ f"Models empty for two defaults."
+ f"
",
+ elem_classes="view-help-host",
+ )
+ with gr.Row(equal_height=False, elem_classes="compare-controls"):
+ prompt_count = gr.Slider(
+ minimum=1,
+ maximum=MAX_COMPARE_PROMPTS,
+ value=DEFAULT_COMPARE_PROMPTS,
+ step=1,
+ label="Prompts to show",
+ container=False,
+ show_reset_button=False,
+ scale=1,
+ min_width=180,
+ elem_classes="compare-prompt-count",
+ )
+ shuffle_button = gr.Button(
+ "Shuffle prompts",
+ variant="primary",
+ scale=0,
+ min_width=140,
+ elem_classes="compare-shuffle",
+ )
+ gallery = gr.HTML(
+ value=_samples_html(
+ initial_samples, [], DEFAULT_COMPARE_PROMPTS, seed=0
+ ),
+ elem_classes="compare-gallery",
+ )
+ seed_state = gr.State(0)
+
+ with gr.TabItem("About", id=TAB_ABOUT) as about_tab:
+ render_about()
+
+ def _synced_filters(dataset_id, metric_id, models, *, clear_metric=False):
+ if clear_metric:
+ metric_id = []
+ else:
+ metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
+ model_choices = _model_choices(datasets, dataset_id)
+ model_values = set(_model_choice_values(model_choices))
+ models = [model for model in (models or []) if model in model_values]
+ metric_choices = _metric_dropdown_choices(datasets, metrics, dataset_id)
return (
- gr.Column(visible=True),
- *[gr.Column(visible=False) for _ in detail_entries],
+ dataset_id,
+ metric_id,
+ models,
+ gr.update(
+ choices=metric_choices,
+ value=_metric_dropdown_value(metric_id),
+ ),
+ gr.update(choices=model_choices, value=models),
)
- def show_detail(selected_id):
+ def _leaderboard_extras(data, platform_value, owner_value, optimized_value):
+ platform_choices = _filter_choices(data, "Platform")
+ owner_choices = _filter_choices(data, "Endpoint Owner")
+ optimized_choices = _filter_choices(data, "Optimized")
+ platform_value = [
+ value for value in (platform_value or []) if value in platform_choices
+ ]
+ owner_value = [
+ value for value in (owner_value or []) if value in owner_choices
+ ]
+ optimized_value = [
+ value for value in (optimized_value or []) if value in optimized_choices
+ ]
return (
- gr.Column(visible=False),
- *[
- gr.Column(visible=(benchmark_id == selected_id))
- for benchmark_id, _, _ in detail_entries
- ],
+ gr.update(
+ choices=platform_choices,
+ value=platform_value,
+ visible=bool(platform_choices),
+ ),
+ gr.update(
+ choices=owner_choices,
+ value=owner_value,
+ visible=bool(owner_choices),
+ ),
+ gr.update(
+ choices=optimized_choices,
+ value=optimized_value,
+ visible=bool(optimized_choices),
+ ),
+ platform_value,
+ owner_value,
+ optimized_value,
+ gr.update(
+ visible=bool(
+ platform_choices or owner_choices or optimized_choices
+ )
+ ),
)
- for benchmark_id, button in open_buttons:
- button.click(
- lambda selected_id=benchmark_id: show_detail(selected_id),
- outputs=nav_outputs,
+ def _content_flags(tab):
+ return {
+ "include_leaderboard": tab == TAB_LEADERBOARDS,
+ "include_pareto": tab == TAB_PARETO,
+ "include_samples": tab == TAB_SAMPLES,
+ }
+
+ def _commit_state(
+ view_state,
+ dataset_id,
+ metric_id,
+ models,
+ tab,
+ flags,
+ extras=None,
+ ):
+ prev = dict(view_state or {})
+ extras = extras or {}
+ return {
+ "dataset_id": dataset_id,
+ "metric_id": metric_id,
+ "models": list(models or []),
+ "current_tab": tab,
+ "platform": list(
+ extras.get("platform", prev.get("platform") or [])
+ ),
+ "owner": list(extras.get("owner", prev.get("owner") or [])),
+ "optimized": list(
+ extras.get("optimized", prev.get("optimized") or [])
+ ),
+ "price_scales": _normalize_pareto_scales(prev.get("price_scales")),
+ "time_scales": _normalize_pareto_scales(prev.get("time_scales")),
+ "stale": {
+ TAB_LEADERBOARDS: not flags["include_leaderboard"],
+ TAB_PARETO: not flags["include_pareto"],
+ TAB_SAMPLES: not flags["include_samples"],
+ },
+ }
+
+ def _save_leaderboard_filters(
+ view_state, platform_value, owner_value, optimized_value
+ ):
+ view_state["platform"] = list(platform_value or [])
+ view_state["owner"] = list(owner_value or [])
+ view_state["optimized"] = list(optimized_value or [])
+ return view_state
+
+ def _restore_leaderboard_filters(
+ view_state, platform_value, owner_value, optimized_value
+ ):
+ stored_platform = (view_state or {}).get("platform") or []
+ stored_owner = (view_state or {}).get("owner") or []
+ stored_optimized = (view_state or {}).get("optimized") or []
+ view = resolve_view(
+ datasets,
+ metrics,
+ (view_state or {}).get("dataset_id"),
+ (view_state or {}).get("metric_id"),
+ )
+ extras = _leaderboard_extras(
+ view["data"] if view else None,
+ stored_platform,
+ stored_owner,
+ stored_optimized,
+ )
+ platform_update = extras[0] if list(platform_value or []) != extras[3] else gr.skip()
+ owner_update = extras[1] if list(owner_value or []) != extras[4] else gr.skip()
+ optimized_update = extras[2] if list(optimized_value or []) != extras[5] else gr.skip()
+ return (
+ extras[6],
+ platform_update,
+ owner_update,
+ optimized_update,
+ extras[3],
+ extras[4],
+ extras[5],
)
- for _, _, back_button in detail_entries:
- back_button.click(show_catalogue, outputs=nav_outputs)
+ def _views(
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ *,
+ include_leaderboard=True,
+ include_pareto=False,
+ include_samples=False,
+ price_scales=None,
+ time_scales=None,
+ ):
+ view = resolve_view(datasets, metrics, dataset_id, metric_id)
+ data = view["data"]
+ if include_leaderboard:
+ note = _leaderboard_intro_markdown(view.get("note"))
+ ranking_html = _leaderboard_html(
+ _filter_leaderboard(
+ _assign_leaderboard_ranks(data, view["score_column"]),
+ platform_value or [],
+ owner_value or [],
+ optimized_value or [],
+ models=models,
+ ),
+ view["columns"],
+ )
+ else:
+ note = gr.skip()
+ ranking_html = gr.skip()
+ if include_pareto:
+ pareto_data = _filter_leaderboard(data, [], [], [], models=models)
+ pareto_updates = _pareto_slot_updates(
+ pareto_data,
+ view["score_columns"],
+ price_scales=price_scales,
+ time_scales=time_scales,
+ )
+ else:
+ pareto_updates = _pareto_skip_updates()
+ if include_samples:
+ samples = view.get("samples")
+ samples_html = _samples_html(
+ samples,
+ models,
+ int(num_prompts or DEFAULT_COMPARE_PROMPTS),
+ int(seed or 0),
+ )
+ samples_visible = gr.update(visible=bool(samples))
+ else:
+ samples_html = gr.skip()
+ samples_visible = gr.skip()
+ return (
+ note,
+ ranking_html,
+ *pareto_updates,
+ samples_html,
+ samples_visible,
+ )
- return show_catalogue, nav_outputs
+ def _apply_filter_change(
+ source,
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ ):
+ view_state = dict(view_state or {})
+ tab = view_state.get("current_tab") or TAB_LEADERBOARDS
+ selected_raw = _normalize_metric_ids(metric_id)
+ incoming_models = list(models or [])
+ dataset_changed = source == "dataset" and dataset_id != view_state.get(
+ "dataset_id"
+ )
+ if source == "dataset":
+ synced = _synced_filters(
+ dataset_id, metric_id, models, clear_metric=dataset_changed
+ )
+ dataset_id, metric_id, models = synced[:3]
+ metric_update, models_update = synced[3], synced[4]
+ else:
+ metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
+ if source == "models":
+ allowed = set(
+ _model_choice_values(_model_choices(datasets, dataset_id))
+ )
+ models = [model for model in incoming_models if model in allowed]
+ models_update = (
+ gr.update(value=models)
+ if models != incoming_models
+ else gr.skip()
+ )
+ else:
+ models = incoming_models
+ models_update = gr.skip()
+ metric_update = (
+ gr.update(
+ choices=_metric_dropdown_choices(datasets, metrics, dataset_id),
+ value=_metric_dropdown_value(metric_id),
+ )
+ if source == "metric" and ALL_METRICS_ID in selected_raw
+ else gr.skip()
+ )
-def _pareto_frontier_mask(x_values, scores):
- """True for non-dominated points when maximizing score and minimizing x."""
- n = len(x_values)
- mask = [True] * n
- for i in range(n):
- for j in range(n):
- if i == j:
- continue
- better_or_equal = x_values[j] <= x_values[i] and scores[j] >= scores[i]
- strictly_better = x_values[j] < x_values[i] or scores[j] > scores[i]
- if better_or_equal and strictly_better:
- mask[i] = False
- break
- return mask
+ unchanged = _applied_key(view_state) == _selection_key(
+ dataset_id, metric_id, models
+ )
+ if (
+ not dataset_changed
+ and not (source == "metric" and ALL_METRICS_ID in selected_raw)
+ and unchanged
+ ):
+ return None
+ selected_tab = tab
+ extras = (
+ list(platform_value or []),
+ list(owner_value or []),
+ list(optimized_value or []),
+ )
+ extra_updates = None
+ can_pareto = _dataset_has_pareto(datasets, dataset_id)
+ can_samples = _dataset_has_samples(datasets, dataset_id)
+ if source == "dataset":
+ if tab == TAB_SAMPLES and not can_samples:
+ selected_tab = TAB_LEADERBOARDS
+ elif tab == TAB_PARETO and not can_pareto:
+ selected_tab = TAB_LEADERBOARDS
+ view = resolve_view(datasets, metrics, dataset_id, metric_id)
+ extra_updates = _leaderboard_extras(
+ view["data"] if view else None,
+ platform_value,
+ owner_value,
+ optimized_value,
+ )
+ extras = extra_updates[3:6]
-def _build_pareto_figure(
- data,
- score_column,
- x_column,
- x_title,
- x_hover_prefix="",
- x_hover_suffix="",
-):
- scatter = (
- data[["Model", score_column, x_column]]
- .dropna()
- .copy()
- .reset_index(drop=True)
- )
- if scatter.empty:
- return None
+ flags = _content_flags(selected_tab)
+ extras_payload = (
+ {
+ "platform": extras[0],
+ "owner": extras[1],
+ "optimized": extras[2],
+ }
+ if selected_tab == TAB_LEADERBOARDS
+ else {}
+ )
+ return {
+ "dataset_id": dataset_id,
+ "metric_update": metric_update,
+ "models_update": models_update,
+ "selected_tab": selected_tab,
+ "tab": tab,
+ "extra_updates": extra_updates,
+ "can_pareto": can_pareto,
+ "can_samples": can_samples,
+ "views": _views(
+ dataset_id,
+ metric_id,
+ models,
+ extras[0],
+ extras[1],
+ extras[2],
+ num_prompts,
+ seed,
+ price_scales=view_state.get("price_scales"),
+ time_scales=view_state.get("time_scales"),
+ **flags,
+ ),
+ "state": _commit_state(
+ view_state,
+ dataset_id,
+ metric_id,
+ models,
+ selected_tab,
+ flags,
+ extras=extras_payload,
+ ),
+ }
+
+ def on_dataset(
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ ):
+ result = _apply_filter_change(
+ "dataset",
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ )
+ if result is None:
+ return _skip_all(len(dataset_outputs))
+ extras = result["extra_updates"]
+ return (
+ _dataset_dropdown_update(
+ datasets, result["selected_tab"], result["dataset_id"]
+ ),
+ result["metric_update"],
+ result["models_update"],
+ extras[6],
+ extras[0],
+ extras[1],
+ extras[2],
+ *result["views"],
+ gr.update(interactive=result["can_pareto"]),
+ gr.update(interactive=result["can_samples"]),
+ gr.update(selected=result["selected_tab"])
+ if result["selected_tab"] != result["tab"]
+ else gr.skip(),
+ result["state"],
+ )
- x_values = scatter[x_column].astype(float).tolist()
- scores = scatter[score_column].astype(float).tolist()
- on_frontier = _pareto_frontier_mask(x_values, scores)
+ def on_metric(
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ ):
+ result = _apply_filter_change(
+ "metric",
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ )
+ if result is None:
+ return _skip_all(len(metric_outputs))
+ return (result["metric_update"], *result["views"], result["state"])
+
+ def on_models(
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ ):
+ result = _apply_filter_change(
+ "models",
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ )
+ if result is None:
+ return _skip_all(len(models_outputs))
+ return (result["models_update"], *result["views"], result["state"])
+
+ def on_tab_select(
+ tab,
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ ):
+ view_state = dict(view_state or {})
+ prev_tab = view_state.get("current_tab") or TAB_LEADERBOARDS
+ requested_tab = tab
+ if tab == TAB_SAMPLES and not _dataset_has_samples(datasets, dataset_id):
+ tab = TAB_LEADERBOARDS
+ elif tab == TAB_PARETO and not _dataset_has_pareto(datasets, dataset_id):
+ tab = TAB_LEADERBOARDS
+ if prev_tab == TAB_LEADERBOARDS:
+ _save_leaderboard_filters(
+ view_state,
+ platform_value,
+ owner_value,
+ optimized_value,
+ )
+ dataset_update = _dataset_dropdown_update(datasets, tab, dataset_id)
+ metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
+ models = list(models or [])
+ view_state["current_tab"] = tab
+ view_state["dataset_id"] = dataset_id
+ view_state["metric_id"] = metric_id
+ view_state["models"] = models
+ show_filters = tab != TAB_ABOUT
+ show_metric = tab in (TAB_LEADERBOARDS, TAB_PARETO)
+ was_filters = prev_tab != TAB_ABOUT
+ was_metric = prev_tab in (TAB_LEADERBOARDS, TAB_PARETO)
+ filters_vis = (
+ gr.update(visible=show_filters)
+ if show_filters != was_filters
+ else gr.skip()
+ )
+ metric_vis = (
+ gr.update(visible=show_metric)
+ if show_metric != was_metric
+ else gr.skip()
+ )
+ if tab == TAB_LEADERBOARDS:
+ restored = _restore_leaderboard_filters(
+ view_state,
+ platform_value,
+ owner_value,
+ optimized_value,
+ )
+ platform_value = restored[4]
+ owner_value = restored[5]
+ optimized_value = restored[6]
+ lb_filters = restored[:4]
+ else:
+ lb_filters = _skip_all(4)
+ stale = dict(view_state.get("stale") or {})
+ chrome = (
+ filters_vis,
+ dataset_update,
+ metric_vis,
+ gr.skip(),
+ *lb_filters,
+ )
+ tab_select = (
+ gr.update(selected=tab)
+ if tab != requested_tab
+ else gr.skip()
+ )
+ if tab == TAB_ABOUT or not stale.get(tab, True):
+ return (
+ *chrome,
+ *_skip_all(len(view_outputs)),
+ tab_select,
+ view_state,
+ )
+ flags = _content_flags(tab)
+ views = _views(
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ price_scales=view_state.get("price_scales"),
+ time_scales=view_state.get("time_scales"),
+ **flags,
+ )
+ stale[tab] = False
+ view_state["stale"] = stale
+ return (*chrome, *views, tab_select, view_state)
+
+ def on_leaderboard_filters(
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ view_state,
+ ):
+ view_state = dict(view_state or {})
+ _save_leaderboard_filters(
+ view_state,
+ platform_value,
+ owner_value,
+ optimized_value,
+ )
+ view = resolve_view(datasets, metrics, dataset_id, metric_id)
+ return (
+ _leaderboard_html(
+ _filter_leaderboard(
+ _assign_leaderboard_ranks(view["data"], view["score_column"]),
+ platform_value or [],
+ owner_value or [],
+ optimized_value or [],
+ models=models,
+ ),
+ view["columns"],
+ ),
+ view_state,
+ )
- dominated = scatter.loc[[not flag for flag in on_frontier]]
- frontier = scatter.loc[on_frontier].sort_values(x_column)
- hover = (
- "
%{text}"
- f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"
- "
Score: %{y:.4f}
"
- )
+ def on_samples_controls(dataset_id, models, num_prompts, seed):
+ view = resolve_view(datasets, metrics, dataset_id, None)
+ return _samples_html(
+ view.get("samples") if view else None,
+ models,
+ int(num_prompts or DEFAULT_COMPARE_PROMPTS),
+ int(seed or 0),
+ )
- fig = go.Figure()
- if not dominated.empty:
- fig.add_trace(
- go.Scatter(
- x=dominated[x_column],
- y=dominated[score_column],
- mode="markers",
- name="Below frontier",
- text=dominated["Model"],
- hovertemplate=hover,
- marker={
- "size": 9,
- "color": "#c4b5fd",
- "opacity": 0.75,
- "line": {"width": 0},
- },
- )
+ def on_shuffle(dataset_id, models, num_prompts, seed):
+ next_seed = int(seed or 0) + 1
+ view = resolve_view(datasets, metrics, dataset_id, None)
+ return next_seed, _samples_html(
+ view.get("samples") if view else None,
+ models,
+ int(num_prompts or DEFAULT_COMPARE_PROMPTS),
+ next_seed,
)
- if not frontier.empty:
- fig.add_trace(
- go.Scatter(
- x=frontier[x_column],
- y=frontier[score_column],
- mode="lines+markers",
- name="On frontier",
- text=frontier["Model"],
- hovertemplate=hover,
- line={"color": "#7c3aed", "width": 2.5},
- marker={
- "size": 12,
- "color": "#db2777",
- "line": {"width": 1.5, "color": "#7c3aed"},
- },
+
+ def _on_pareto_plot_scale(slot_index, axis):
+ def handler(dataset_id, metric_id, models, scale, view_state):
+ view_state = dict(view_state or {})
+ price_scales = _normalize_pareto_scales(
+ view_state.get("price_scales")
+ )
+ time_scales = _normalize_pareto_scales(
+ view_state.get("time_scales")
+ )
+ if axis == "price":
+ if price_scales[slot_index] == scale:
+ return gr.skip(), gr.skip(), gr.skip()
+ price_scales[slot_index] = scale
+ else:
+ if time_scales[slot_index] == scale:
+ return gr.skip(), gr.skip(), gr.skip()
+ time_scales[slot_index] = scale
+ view_state["price_scales"] = price_scales
+ view_state["time_scales"] = time_scales
+ master_scale = _pareto_master_scale_update(
+ price_scales, time_scales
+ )
+ view = resolve_view(datasets, metrics, dataset_id, metric_id)
+ score_columns = [
+ column for column in (view["score_columns"] or []) if column
+ ]
+ if slot_index >= len(score_columns):
+ return gr.skip(), master_scale, view_state
+ data = _filter_leaderboard(
+ view["data"], [], [], [], models=list(models or [])
+ )
+ price_fig, _, time_fig, _ = _pareto_pair(
+ data,
+ score_columns[slot_index],
+ latency_scale=time_scales[slot_index],
+ price_scale=price_scales[slot_index],
)
+ fig = price_fig if axis == "price" else time_fig
+ return _pareto_plot_update(fig), master_scale, view_state
+
+ handler.__name__ = f"on_pareto_{axis}_scale_{slot_index}"
+ return handler
+
+ def on_pareto_all_scale(dataset_id, metric_id, models, scale, view_state):
+ if scale not in _PARETO_SCALE_VALUES:
+ return (*_skip_all(MAX_PARETO_METRICS * 4), gr.skip())
+ view_state = dict(view_state or {})
+ scales = _uniform_pareto_scales(scale)
+ if (
+ _normalize_pareto_scales(view_state.get("price_scales")) == scales
+ and _normalize_pareto_scales(view_state.get("time_scales")) == scales
+ ):
+ return (*_skip_all(MAX_PARETO_METRICS * 4), gr.skip())
+ view_state["price_scales"] = scales
+ view_state["time_scales"] = scales
+ view = resolve_view(datasets, metrics, dataset_id, metric_id)
+ data = _filter_leaderboard(
+ view["data"], [], [], [], models=list(models or [])
+ )
+ return (
+ *_pareto_all_scale_updates(data, view["score_columns"], scale),
+ view_state,
)
- score_label = _display_label(score_column)
- fig.update_layout(
- title=None,
- xaxis_title=x_title,
- yaxis_title=score_label,
- autosize=True,
- height=420,
- margin={"l": 56, "r": 28, "t": 28, "b": 80},
- legend={
- "orientation": "h",
- "yanchor": "top",
- "y": -0.24,
- "xanchor": "center",
- "x": 0.5,
- "bgcolor": "rgba(0,0,0,0)",
- "font": {"color": "#e9d5ff", "size": 12},
- },
- # Dark-theme plot: soft purple panel + light text (readable, not a white flash).
- plot_bgcolor="#1e1b4b",
- paper_bgcolor="#17153b",
- font={"color": "#e9d5ff", "size": 13},
+ def _on_tab(tab):
+ def handler(
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ ):
+ return on_tab_select(
+ tab,
+ dataset_id,
+ metric_id,
+ models,
+ platform_value,
+ owner_value,
+ optimized_value,
+ num_prompts,
+ seed,
+ view_state,
+ )
+
+ handler.__name__ = f"on_tab_{tab}"
+ return handler
+
+ view_state = gr.State(
+ {
+ "dataset_id": default_dataset_id,
+ "metric_id": None,
+ "models": [],
+ "current_tab": TAB_LEADERBOARDS,
+ "platform": [],
+ "owner": [],
+ "optimized": [],
+ "price_scales": _default_pareto_scales(),
+ "time_scales": _default_pareto_scales(),
+ "stale": {
+ TAB_LEADERBOARDS: False,
+ TAB_PARETO: True,
+ TAB_SAMPLES: False,
+ },
+ }
)
- axis_font = {"color": "#f3e8ff", "size": 13}
- tick_font = {"color": "#ddd6fe", "size": 12}
- fig.update_xaxes(
- showgrid=True,
- gridcolor="rgba(167, 139, 250, 0.28)",
- zeroline=False,
- title_font=axis_font,
- tickfont=tick_font,
- color="#e9d5ff",
+ pareto_outputs = [
+ pareto_dataset_note,
+ *[
+ component
+ for slot_group, slot_title, slot_note, slot_layout, slot_price_col, slot_price, slot_price_scale, slot_time_col, slot_time, slot_time_scale in pareto_slots
+ for component in (
+ slot_group,
+ slot_title,
+ slot_note,
+ slot_layout,
+ slot_price_col,
+ slot_price,
+ slot_time_col,
+ slot_time,
+ )
+ ],
+ ]
+ pareto_all_scale_outputs = [
+ *[
+ slot_price
+ for _, _, _, _, _, slot_price, _, _, _, _ in pareto_slots
+ ],
+ *[
+ slot_time
+ for _, _, _, _, _, _, _, _, slot_time, _ in pareto_slots
+ ],
+ *[
+ slot_price_scale
+ for _, _, _, _, _, _, slot_price_scale, _, _, _ in pareto_slots
+ ],
+ *[
+ slot_time_scale
+ for _, _, _, _, _, _, _, _, _, slot_time_scale in pareto_slots
+ ],
+ ]
+ view_inputs = [
+ platform,
+ owner,
+ optimized,
+ prompt_count,
+ seed_state,
+ view_state,
+ ]
+ view_outputs = [
+ lb_note,
+ ranking,
+ *pareto_outputs,
+ gallery,
+ samples_panel,
+ ]
+ filter_inputs = [dataset_dd, metric_dd, models_dd, *view_inputs]
+
+ dataset_outputs = [
+ dataset_dd,
+ metric_dd,
+ models_dd,
+ lb_controls,
+ platform,
+ owner,
+ optimized,
+ *view_outputs,
+ pp_tab,
+ sm_tab,
+ main_tabs,
+ view_state,
+ ]
+ dataset_dd.change(
+ on_dataset,
+ inputs=filter_inputs,
+ outputs=dataset_outputs,
+ **_VIEW_EVENTS,
)
- fig.update_yaxes(
- showgrid=True,
- gridcolor="rgba(167, 139, 250, 0.28)",
- zeroline=False,
- title_font=axis_font,
- tickfont=tick_font,
- color="#e9d5ff",
+
+ metric_outputs = [
+ metric_dd,
+ *view_outputs,
+ view_state,
+ ]
+ metric_dd.change(
+ on_metric,
+ inputs=filter_inputs,
+ outputs=metric_outputs,
+ **_VIEW_EVENTS,
)
- return fig
+ models_outputs = [
+ models_dd,
+ *view_outputs,
+ view_state,
+ ]
+ models_dd.change(
+ on_models,
+ inputs=filter_inputs,
+ outputs=models_outputs,
+ **_VIEW_EVENTS,
+ )
-def render_benchmark_graphs(benchmark):
- data = benchmark["data"]
- score_columns = [
- column
- for column in (benchmark.get("score_columns") or [])
- if column in data.columns
+ tab_outputs = [
+ filters_host,
+ dataset_dd,
+ metric_dd,
+ models_dd,
+ lb_controls,
+ platform,
+ owner,
+ optimized,
+ *view_outputs,
+ main_tabs,
+ view_state,
]
- overall_column = benchmark.get("overall_column")
-
- if not score_columns and overall_column and overall_column in data.columns:
- score_columns = [overall_column]
-
- if not score_columns:
- gr.Markdown("No score data is available yet.")
- return
-
- # Pareto every displayed quality metric vs price.
- # Skip only synthetic aggregates (e.g. OneIG mean), not real sort keys like Datapoint Elo.
- pareto_skip = {
- "Model",
- "Platform",
- "Endpoint Owner",
- "Optimized",
- "URL",
- "Rank",
- "Median Generation Time (s)",
- "Min Generation Time (s)",
- "Price / Image (USD)",
- "Evaluation Date (UTC)",
- "Date",
- "Raw Win Rate",
- "OneIG Overall Score",
- }
+ for tab, tab_item in (
+ (TAB_LEADERBOARDS, lb_tab),
+ (TAB_PARETO, pp_tab),
+ (TAB_SAMPLES, sm_tab),
+ (TAB_ABOUT, about_tab),
+ ):
+ tab_item.select(
+ _on_tab(tab),
+ inputs=filter_inputs,
+ outputs=tab_outputs,
+ show_progress="hidden",
+ )
- display_columns = benchmark.get("columns") or []
- pareto_columns = []
- for column in [*score_columns, *display_columns]:
- if (
- column in data.columns
- and column not in pareto_skip
- and column not in pareto_columns
- and pd.api.types.is_numeric_dtype(data[column])
- ):
- pareto_columns.append(column)
-
- price_column = "Price / Image (USD)"
- time_column = "Min Generation Time (s)"
- price_figures = []
- time_figures = []
- for plot_column in pareto_columns:
- if price_column in data.columns:
- price_fig = _build_pareto_figure(
- data,
- plot_column,
- x_column=price_column,
- x_title="Price per image (USD)",
- x_hover_prefix="$",
- )
- if price_fig is not None:
- price_figures.append((plot_column, price_fig))
- if time_column in data.columns:
- time_fig = _build_pareto_figure(
- data,
- plot_column,
- x_column=time_column,
- x_title="Min generation time (s)",
- x_hover_suffix="s",
- )
- if time_fig is not None:
- time_figures.append((plot_column, time_fig))
-
- if price_figures or time_figures:
- gr.Markdown(
- "### Pareto frontiers\n\n"
- "
"
- "Pink = on the frontier (lower cost or time at the same or better score). "
- "Light purple = below the frontier."
- ""
+ for component in (platform, owner, optimized):
+ component.change(
+ on_leaderboard_filters,
+ inputs=[
+ dataset_dd,
+ metric_dd,
+ models_dd,
+ platform,
+ owner,
+ optimized,
+ view_state,
+ ],
+ outputs=[ranking, view_state],
+ show_progress="hidden",
)
- with gr.Row(equal_height=False):
- with gr.Column(scale=1, min_width=320):
- gr.Markdown("#### Price vs score")
- if not price_figures:
- gr.Markdown("_No price data available._")
- for plot_column, pareto_fig in price_figures:
- gr.Markdown(f"**{_display_label(plot_column)}**")
- gr.Plot(
- value=pareto_fig,
- show_label=False,
- elem_classes="pareto-plot",
- )
- with gr.Column(scale=1, min_width=320):
- gr.Markdown("#### Min generation time vs score")
- if not time_figures:
- gr.Markdown("_No min generation time data available._")
- for plot_column, pareto_fig in time_figures:
- gr.Markdown(f"**{_display_label(plot_column)}**")
- gr.Plot(
- value=pareto_fig,
- show_label=False,
- elem_classes="pareto-plot",
- )
+ pareto_all_scale.change(
+ on_pareto_all_scale,
+ inputs=[
+ dataset_dd,
+ metric_dd,
+ models_dd,
+ pareto_all_scale,
+ view_state,
+ ],
+ outputs=[*pareto_all_scale_outputs, view_state],
+ **_VIEW_EVENTS,
+ )
+
+ for slot_index, (
+ _,
+ _,
+ _,
+ _,
+ _,
+ slot_price,
+ slot_price_scale,
+ _,
+ slot_time,
+ slot_time_scale,
+ ) in enumerate(pareto_slots):
+ slot_price_scale.change(
+ _on_pareto_plot_scale(slot_index, "price"),
+ inputs=[
+ dataset_dd,
+ metric_dd,
+ models_dd,
+ slot_price_scale,
+ view_state,
+ ],
+ outputs=[slot_price, pareto_all_scale, view_state],
+ **_VIEW_EVENTS,
+ )
+ slot_time_scale.change(
+ _on_pareto_plot_scale(slot_index, "time"),
+ inputs=[
+ dataset_dd,
+ metric_dd,
+ models_dd,
+ slot_time_scale,
+ view_state,
+ ],
+ outputs=[slot_time, pareto_all_scale, view_state],
+ **_VIEW_EVENTS,
+ )
+
+ prompt_count.change(
+ on_samples_controls,
+ inputs=[dataset_dd, models_dd, prompt_count, seed_state],
+ outputs=gallery,
+ show_progress="hidden",
+ )
+ shuffle_button.click(
+ on_shuffle,
+ inputs=[dataset_dd, models_dd, prompt_count, seed_state],
+ outputs=[seed_state, gallery],
+ show_progress="hidden",
+ )
def render_about():
- with gr.Row():
- with gr.Column():
- gr.Markdown(ABOUT_OVERVIEW_CONTENT)
- with gr.Column():
- gr.Markdown(ABOUT_DETAILS_CONTENT)
+ with gr.Row(elem_classes="about-layout", equal_height=False):
+ with gr.Column(scale=1, min_width=340, elem_classes="about-col"):
+ gr.Markdown(ABOUT_OVERVIEW_CONTENT, elem_classes="about-copy")
+ with gr.Column(scale=1, min_width=340, elem_classes="about-col"):
+ gr.Markdown(ABOUT_DETAILS_CONTENT, elem_classes="about-copy")
def render_footer():
- gr.HTML(COMMUNITY_CONTENT)
- with gr.Accordion("Citation", open=False):
+ gr.HTML(FOOTER_CONTENT, padding=False, elem_classes="page-footer-host")
+ with gr.Accordion(
+ "Citation",
+ open=False,
+ elem_classes="citation-accordion",
+ ):
gr.Markdown(CITATION_CONTENT)
+