Minette Kaunismäki commited on
Commit
4b9f11d
·
1 Parent(s): 0fa1882
Files changed (2) hide show
  1. app.py +188 -67
  2. ui.py +783 -641
app.py CHANGED
@@ -14,11 +14,9 @@ import gradio as gr
14
  import pandas as pd
15
 
16
  from ui import (
17
- render_about,
18
- render_benchmarks,
19
  render_footer,
20
  render_header,
21
- render_home,
22
  )
23
 
24
  # Visual tokens from https://www.pruna.ai/ and https://playground.pruna.ai/
@@ -165,7 +163,8 @@ body, .gradio-container {
165
  .benchmark-catalogue,
166
  .benchmark-detail,
167
  .benchmark-panel,
168
- .leaderboard-controls {
 
169
  max-width: 100% !important;
170
  }
171
 
@@ -388,7 +387,8 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
388
  .benchmark-catalogue,
389
  .benchmark-detail,
390
  .benchmark-panel,
391
- .leaderboard-controls {
 
392
  min-width: 0 !important;
393
  max-width: 100% !important;
394
  }
@@ -471,12 +471,17 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
471
  }
472
  .leaderboard-controls,
473
  .leaderboard-controls.row,
474
- .leaderboard-controls .form {
 
 
 
475
  flex-direction: column !important;
476
  align-items: stretch !important;
477
  }
478
  .leaderboard-controls > div,
479
- .leaderboard-controls .form > div {
 
 
480
  flex: 1 1 auto !important;
481
  width: 100% !important;
482
  max-width: 100% !important;
@@ -568,7 +573,58 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
568
  }
569
  }
570
 
571
- /* —— Benchmark view menu (Leaderboard / Graphs / Compare) —— */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
572
  .benchmark-back-btn,
573
  .benchmark-back-btn.block,
574
  .benchmark-back-btn .padded {
@@ -580,6 +636,10 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
580
  background: transparent !important;
581
  box-shadow: none !important;
582
  }
 
 
 
 
583
  .benchmark-back-btn button,
584
  .benchmark-back-btn button.secondary,
585
  .benchmark-back-btn button.lg,
@@ -603,6 +663,7 @@ button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !import
603
  letter-spacing: -0.01em !important;
604
  line-height: 1.4 !important;
605
  }
 
606
  .benchmark-back-btn button:hover {
607
  color: var(--pruna-link) !important;
608
  background: transparent !important;
@@ -2227,6 +2288,7 @@ def load_qwen_combined_dataframe(path):
2227
  "P-Judge Overall",
2228
  "Rapidata Elo",
2229
  "Datapoint Elo",
 
2230
  ]:
2231
  if column in df.columns:
2232
  df[column] = pd.to_numeric(df[column], errors="coerce")
@@ -2279,6 +2341,7 @@ oneig_display_columns = [
2279
  "Endpoint Owner",
2280
  "Model",
2281
  "Optimized",
 
2282
  *oneig_metric_columns,
2283
  "OneIG Anime Elo",
2284
  "OneIG Human Elo",
@@ -2293,8 +2356,6 @@ oneig_display_columns = [
2293
  ]
2294
  if col in oneig_df.columns
2295
  ]
2296
- # Top-level Leaderboard tab uses the same OneIG table.
2297
- display_columns = oneig_display_columns
2298
 
2299
  oneig_combined_dir = _resolve_data_path(
2300
  data_dir / "oneig_combined",
@@ -2310,87 +2371,152 @@ qwen_path = _resolve_data_path(
2310
  )
2311
 
2312
  qwen_df = load_qwen_combined_dataframe(qwen_path)
2313
- qwen_score_columns = [
2314
  col
2315
  for col in [
2316
- "P-Judge Overall",
2317
  "Datapoint Elo",
2318
  "Rapidata Elo",
 
 
 
 
 
2319
  ]
2320
  if col in qwen_df.columns
2321
  ]
2322
- qwen_display_columns = [
2323
  col
2324
  for col in [
2325
  "Model",
2326
- *qwen_score_columns,
2327
- "Raw Win Rate",
2328
  "Median Generation Time (s)",
2329
  "Min Generation Time (s)",
2330
  "Price / Image (USD)",
2331
  ]
2332
  if col in qwen_df.columns
2333
  ]
2334
- qwen_overall_column = (
2335
- "Datapoint Elo"
2336
- if "Datapoint Elo" in qwen_df.columns
2337
- else (qwen_score_columns[0] if qwen_score_columns else None)
2338
- )
2339
 
2340
  oneig_samples = load_sample_comparison_data(oneig_combined_dir)
2341
  qwen_samples = load_sample_comparison_data(qwen_combined_dir)
2342
 
2343
- # Dataset-first catalogue: each card is a prompt suite; metrics are leaderboard columns.
2344
- benchmarks = [
2345
  {
2346
- "id": "oneig",
2347
- "title": "OneIG Alignment",
2348
- "emoji": "🎯",
2349
- "card_description": (
2350
- "Anime/stylization, portrait, and general-object alignment prompts: "
2351
- "alignment scores and Datapoint Elo (not the full OneIG suite), plus "
2352
- "side-by-side generations."
2353
- ),
2354
- "intro": (
2355
- "OneIG Alignment covers the alignment slice of OneIG (not every OneIG "
2356
- "dimension). The leaderboard shows category alignment scores and Datapoint "
2357
- "Elo columns; Compare samples uses the combined alignment generations."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2358
  ),
 
 
 
 
 
2359
  "data": oneig_df,
2360
  "columns": oneig_display_columns,
2361
- "score_columns": oneig_metric_columns,
2362
- "overall_column": "OneIG Overall Score",
2363
  "note": (
2364
- "> Rankings are ordered by the mean of the available alignment category "
2365
- "scores. Missing category scores are not included in that model's mean."
2366
  ),
2367
  "samples": oneig_samples,
2368
  },
2369
  {
2370
- "id": "qwen_image_bench",
2371
- "title": "Qwen Image Bench",
2372
- "emoji": "🖼️",
2373
- "card_description": (
2374
- "Qwen image-bench prompts with P-Judger (Pruna's judge), Datapoint Elo, "
2375
- "and Rapidata Elo as metric columns, plus combined generations for comparison."
2376
- ),
2377
- "intro": (
2378
- "Qwen Image Bench is a shared prompt suite. The leaderboard joins every "
2379
- "available metric for this benchmark; Compare samples uses the combined "
2380
- "Qwen generations."
2381
- ),
2382
  "data": qwen_df,
2383
- "columns": qwen_display_columns,
2384
- "score_columns": qwen_score_columns,
2385
- "overall_column": qwen_overall_column,
2386
  "note": (
2387
- "> Models are ordered by Datapoint Elo when available. Other metric "
2388
- "columns come from Pruna's P-Judger and the Rapidata evaluation on the "
2389
- "same prompt suite."
2390
  ),
2391
  "samples": qwen_samples,
2392
  },
2393
  ]
 
 
 
 
 
 
 
 
 
 
 
2394
 
2395
 
2396
  custom_head = """
@@ -2615,17 +2741,12 @@ with gr.Blocks(
2615
  head=custom_head,
2616
  ) as demo:
2617
  render_header()
2618
- with gr.Tabs(elem_classes="main-tabs"):
2619
- with gr.TabItem("Home"):
2620
- render_home(benchmarks)
2621
- with gr.TabItem("About"):
2622
- render_about()
2623
- with gr.TabItem("Benchmarks") as benchmarks_tab:
2624
- reset_benchmarks, reset_benchmark_outputs = render_benchmarks(benchmarks)
2625
- benchmarks_tab.select(
2626
- reset_benchmarks,
2627
- outputs=reset_benchmark_outputs,
2628
- )
2629
  render_footer()
2630
 
2631
 
 
14
  import pandas as pd
15
 
16
  from ui import (
 
 
17
  render_footer,
18
  render_header,
19
+ render_image_workspace,
20
  )
21
 
22
  # Visual tokens from https://www.pruna.ai/ and https://playground.pruna.ai/
 
163
  .benchmark-catalogue,
164
  .benchmark-detail,
165
  .benchmark-panel,
166
+ .leaderboard-controls,
167
+ .view-filters {
168
  max-width: 100% !important;
169
  }
170
 
 
387
  .benchmark-catalogue,
388
  .benchmark-detail,
389
  .benchmark-panel,
390
+ .leaderboard-controls,
391
+ .view-filters {
392
  min-width: 0 !important;
393
  max-width: 100% !important;
394
  }
 
471
  }
472
  .leaderboard-controls,
473
  .leaderboard-controls.row,
474
+ .leaderboard-controls .form,
475
+ .view-filters,
476
+ .view-filters.row,
477
+ .view-filters .form {
478
  flex-direction: column !important;
479
  align-items: stretch !important;
480
  }
481
  .leaderboard-controls > div,
482
+ .leaderboard-controls .form > div,
483
+ .view-filters > div,
484
+ .view-filters .form > div {
485
  flex: 1 1 auto !important;
486
  width: 100% !important;
487
  max-width: 100% !important;
 
573
  }
574
  }
575
 
576
+ /* —— Benchmark view menu (Leaderboards / Pareto plots / Compare) —— */
577
+ .view-filters {
578
+ display: flex !important;
579
+ flex-wrap: wrap !important;
580
+ align-items: end !important;
581
+ gap: 10px !important;
582
+ margin: 0 0 8px;
583
+ }
584
+ .view-filters > div {
585
+ min-width: 0 !important;
586
+ }
587
+ .view-filters > .block,
588
+ .view-filters .form,
589
+ .view-filters .block {
590
+ border: none !important;
591
+ background: transparent !important;
592
+ box-shadow: none !important;
593
+ padding: 0 !important;
594
+ --block-border-width: 0 !important;
595
+ }
596
+ .view-filters label {
597
+ color: var(--pruna-text-muted) !important;
598
+ font-size: 0.8rem !important;
599
+ font-weight: 500 !important;
600
+ }
601
+ .view-filters .wrap,
602
+ .view-filters .wrap-inner,
603
+ .view-filters .secondary-wrap {
604
+ min-height: 40px !important;
605
+ border: 1px solid var(--pruna-input-border) !important;
606
+ border-radius: 10px !important;
607
+ background: var(--pruna-input-bg) !important;
608
+ box-shadow: none !important;
609
+ }
610
+ .view-title,
611
+ .view-title.block,
612
+ .view-title .padded {
613
+ border: none !important;
614
+ background: transparent !important;
615
+ box-shadow: none !important;
616
+ padding: 0 !important;
617
+ margin: 0 0 8px !important;
618
+ }
619
+ .view-title h1,
620
+ .view-title .prose h1,
621
+ .gradio-container .view-title h1 {
622
+ margin: 0.35rem 0 0.4rem !important;
623
+ font-size: 1.45rem !important;
624
+ }
625
+ .workspace-back-btn,
626
+ .workspace-back-btn.block,
627
+ .workspace-back-btn .padded,
628
  .benchmark-back-btn,
629
  .benchmark-back-btn.block,
630
  .benchmark-back-btn .padded {
 
636
  background: transparent !important;
637
  box-shadow: none !important;
638
  }
639
+ .workspace-back-btn button,
640
+ .workspace-back-btn button.secondary,
641
+ .workspace-back-btn button.lg,
642
+ .workspace-back-btn button.sm,
643
  .benchmark-back-btn button,
644
  .benchmark-back-btn button.secondary,
645
  .benchmark-back-btn button.lg,
 
663
  letter-spacing: -0.01em !important;
664
  line-height: 1.4 !important;
665
  }
666
+ .workspace-back-btn button:hover,
667
  .benchmark-back-btn button:hover {
668
  color: var(--pruna-link) !important;
669
  background: transparent !important;
 
2288
  "P-Judge Overall",
2289
  "Rapidata Elo",
2290
  "Datapoint Elo",
2291
+ "Benchmark.ai Elo",
2292
  ]:
2293
  if column in df.columns:
2294
  df[column] = pd.to_numeric(df[column], errors="coerce")
 
2341
  "Endpoint Owner",
2342
  "Model",
2343
  "Optimized",
2344
+ "OneIG Overall Score",
2345
  *oneig_metric_columns,
2346
  "OneIG Anime Elo",
2347
  "OneIG Human Elo",
 
2356
  ]
2357
  if col in oneig_df.columns
2358
  ]
 
 
2359
 
2360
  oneig_combined_dir = _resolve_data_path(
2361
  data_dir / "oneig_combined",
 
2371
  )
2372
 
2373
  qwen_df = load_qwen_combined_dataframe(qwen_path)
2374
+ qwen_display_columns = [
2375
  col
2376
  for col in [
2377
+ "Model",
2378
  "Datapoint Elo",
2379
  "Rapidata Elo",
2380
+ "P-Judge Overall",
2381
+ "Raw Win Rate",
2382
+ "Median Generation Time (s)",
2383
+ "Min Generation Time (s)",
2384
+ "Price / Image (USD)",
2385
  ]
2386
  if col in qwen_df.columns
2387
  ]
2388
+ aa_display_columns = [
2389
  col
2390
  for col in [
2391
  "Model",
2392
+ "Benchmark.ai Elo",
 
2393
  "Median Generation Time (s)",
2394
  "Min Generation Time (s)",
2395
  "Price / Image (USD)",
2396
  ]
2397
  if col in qwen_df.columns
2398
  ]
 
 
 
 
 
2399
 
2400
  oneig_samples = load_sample_comparison_data(oneig_combined_dir)
2401
  qwen_samples = load_sample_comparison_data(qwen_combined_dir)
2402
 
2403
+ metrics = [
 
2404
  {
2405
+ "id": "datapoint_elo",
2406
+ "name": "Datapoint ELO - Overall Metric",
2407
+ "column": "Datapoint Elo",
2408
+ },
2409
+ {
2410
+ "id": "rapidata_elo",
2411
+ "name": "Rapidata ELO - Overall Metric",
2412
+ "column": "Rapidata Elo",
2413
+ },
2414
+ {
2415
+ "id": "pjudger",
2416
+ "name": "P-Judger Overall Metric",
2417
+ "column": "P-Judge Overall",
2418
+ },
2419
+ {
2420
+ "id": "alignment_overall",
2421
+ "name": "Alignment - Overall Metric",
2422
+ "column": "OneIG Overall Score",
2423
+ },
2424
+ {
2425
+ "id": "datapoint_elo_anime",
2426
+ "name": "Datapoint ELO - Anime Metric",
2427
+ "column": "OneIG Anime Elo",
2428
+ },
2429
+ {
2430
+ "id": "datapoint_elo_human",
2431
+ "name": "Datapoint ELO - Human Metric",
2432
+ "column": "OneIG Human Elo",
2433
+ },
2434
+ {
2435
+ "id": "datapoint_elo_object",
2436
+ "name": "Datapoint ELO - Object Metric",
2437
+ "column": "OneIG Object Elo",
2438
+ },
2439
+ {
2440
+ "id": "aa_elo",
2441
+ "name": "Artificial Analysis ELO Metric",
2442
+ "column": "Benchmark.ai Elo",
2443
+ },
2444
+ ]
2445
+
2446
+
2447
+ def _metric_ids_for(data, metric_ids):
2448
+ columns = getattr(data, "columns", [])
2449
+ return [
2450
+ metric_id
2451
+ for metric_id in metric_ids
2452
+ if any(metric["id"] == metric_id and metric["column"] in columns for metric in metrics)
2453
+ ]
2454
+
2455
+
2456
+ qwen_metric_ids = _metric_ids_for(
2457
+ qwen_df, ["datapoint_elo", "rapidata_elo", "pjudger"]
2458
+ )
2459
+ oneig_metric_ids = _metric_ids_for(
2460
+ oneig_df,
2461
+ [
2462
+ "alignment_overall",
2463
+ "rapidata_elo",
2464
+ "pjudger",
2465
+ "datapoint_elo_anime",
2466
+ "datapoint_elo_human",
2467
+ "datapoint_elo_object",
2468
+ ],
2469
+ )
2470
+ aa_metric_ids = _metric_ids_for(qwen_df, ["aa_elo"])
2471
+
2472
+ datasets = [
2473
+ {
2474
+ "id": "qwen",
2475
+ "name": "Qwen Image Dataset",
2476
+ "data": qwen_df,
2477
+ "columns": qwen_display_columns,
2478
+ "metric_ids": qwen_metric_ids,
2479
+ "note": (
2480
+ "> Ranked by the selected metric on the Qwen Image Dataset. "
2481
+ "Rapidata Elo is a metric on this dataset, not a dataset of its own."
2482
  ),
2483
+ "samples": qwen_samples,
2484
+ },
2485
+ {
2486
+ "id": "oneig",
2487
+ "name": "OneIG Alignment Dataset",
2488
  "data": oneig_df,
2489
  "columns": oneig_display_columns,
2490
+ "metric_ids": oneig_metric_ids,
 
2491
  "note": (
2492
+ "> Alignment Overall is the mean of the available category scores. "
2493
+ "Missing categories are skipped for that model."
2494
  ),
2495
  "samples": oneig_samples,
2496
  },
2497
  {
2498
+ "id": "artificial_analysis",
2499
+ "name": "Artificial Analysis Dataset",
 
 
 
 
 
 
 
 
 
 
2500
  "data": qwen_df,
2501
+ "columns": aa_display_columns,
2502
+ "metric_ids": aa_metric_ids,
 
2503
  "note": (
2504
+ "> Ranked by Artificial Analysis Elo from the evaluation table."
 
 
2505
  ),
2506
  "samples": qwen_samples,
2507
  },
2508
  ]
2509
+ datasets = [dataset for dataset in datasets if dataset["metric_ids"]]
2510
+
2511
+ DEFAULT_DATASET_ID = next(
2512
+ (dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
2513
+ datasets[0]["id"] if datasets else None,
2514
+ )
2515
+ DEFAULT_METRIC_ID = (
2516
+ "datapoint_elo"
2517
+ if DEFAULT_DATASET_ID == "qwen" and "datapoint_elo" in qwen_metric_ids
2518
+ else (datasets[0]["metric_ids"][0] if datasets else None)
2519
+ )
2520
 
2521
 
2522
  custom_head = """
 
2741
  head=custom_head,
2742
  ) as demo:
2743
  render_header()
2744
+ render_image_workspace(
2745
+ datasets,
2746
+ metrics,
2747
+ DEFAULT_DATASET_ID,
2748
+ DEFAULT_METRIC_ID,
2749
+ )
 
 
 
 
 
2750
  render_footer()
2751
 
2752
 
ui.py CHANGED
@@ -24,18 +24,18 @@ ABOUT_OVERVIEW_CONTENT = """
24
  # About P-Bench
25
 
26
  P-Bench compares **text-to-image models**, including optimized or accelerated
27
- endpoints, on **quality, speed, and price**. Results are split by prompt suite;
28
- there is no single score across P-Bench.
 
29
 
30
  ## How to read it
31
 
32
- 1. Open a **prompt suite** on the Benchmarks tab.
33
- 2. **Leaderboard**: sort by a quality or preference column. Price and generation
34
- time sit in the same table.
35
- 3. **Graphs**: Pareto plots mark models that are not beaten on both higher score
36
  and lower price (or time).
37
- 4. **Compare samples**: the same prompts, side by side. Sample images are
38
- available for OneIG Alignment today.
39
 
40
  ## How a score is made
41
 
@@ -43,20 +43,26 @@ there is no single score across P-Bench.
43
  2. It generates one image per prompt when the run succeeds. Not every model
44
  has every prompt or every metric.
45
  3. Quality is scored automatically (OneIG alignment, P-Judger) and, where
46
- available, by human preference (Datapoint Elo, Rapidata Elo).
 
47
  4. Price per image and generation time are joined from the evaluation table.
48
 
49
- ## Current prompt suites
50
 
51
- ### OneIG Alignment
 
 
 
 
 
52
  Prompt-image **alignment** on anime / stylization, human / portrait, and
53
  general object prompts (100 prompts each). This is the alignment slice of
54
- OneIG, not the full suite. Default rank is the mean of the alignment
55
- categories that exist for that row.
56
 
57
- ### Qwen Image Bench
58
- 100 prompts from the 1,000-prompt Qwen Image Bench set, sampled for coverage
59
- across its fine-grained (L3) categories.
60
  """
61
 
62
  ABOUT_DETAILS_CONTENT = """
@@ -72,16 +78,18 @@ ABOUT_DETAILS_CONTENT = """
72
  - **Datapoint Elo**: human-preference Elo from Datapoint pairwise comparisons.
73
  - **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.
74
  Rapidata rejects prompts over 400 characters, so this Elo is on a subset
75
- of each suite (see Setup).
 
 
76
  - **Generation time**: median and minimum generation time in seconds, as
77
  reported in the evaluation table. This is not a p95, and we do not state
78
  warm vs cold or concurrent load.
79
  - **Price**: USD per image in the evaluation table. We do not state list
80
  price vs amount paid, or whether failed generations are included.
81
 
82
- Scores from different suites or columns are **not interchangeable**. A high
83
  OneIG alignment score is not the same quantity as a Datapoint Elo. Compare
84
- models *within* a column.
85
 
86
  ## Setup
87
 
@@ -89,7 +97,7 @@ models *within* a column.
89
  - **Update policy:** numbers come from evaluation snapshots in the tables,
90
  not a live API poll.
91
  - **Prompt counts:** OneIG Alignment uses the first 100 prompts from each of
92
- the three categories (300 total). Qwen Image Bench uses 100 prompts sampled
93
  from the 1,000-prompt pool for roughly even coverage of its fine-grained
94
  (L3) categories.
95
  - **Generation:** one image per prompt per endpoint when the run exists.
@@ -101,14 +109,14 @@ models *within* a column.
101
  - **Datapoint:** every model pair is compared on every prompt, with 10 votes
102
  per battle.
103
  - **Rapidata:** prompts longer than 400 characters are dropped, leaving 212
104
- OneIG prompts and 85 Qwen Image Bench prompts. 4 votes per pair; about
105
- 26,000 votes on OneIG and 35,000 on Qwen Image Bench.
106
 
107
  ## Limits
108
 
109
  - Empty cells mean that track was not run or not reported for that model.
110
  - Rapidata Elo is not on the full prompt suite, so it is not directly
111
- comparable to Datapoint Elo even on the same benchmark.
112
  - Elo ratings can shift when the comparison pool changes: treat them as
113
  relative rankings for the snapshot, not absolute constants.
114
  - Close scores can be a tie in practice; the table does not show confidence
@@ -171,119 +179,83 @@ def render_header():
171
  )
172
 
173
 
174
- def _top_models(data, score_column, n=3):
175
- if score_column not in data.columns or "Model" not in data.columns:
176
- return []
177
- ranked = (
178
- data[["Model", score_column]]
179
- .dropna(subset=[score_column])
180
- .loc[lambda df: ~df["Model"].astype(str).str.startswith("#")]
181
- .sort_values(score_column, ascending=False)
182
- .head(n)
183
- )
184
- return [
185
- (str(row["Model"]), float(row[score_column]))
186
- for _, row in ranked.iterrows()
187
- ]
188
 
189
 
190
- def _home_highlights(benchmarks):
191
- """Quality leaders per suite: more relevant than cheapest/fastest outliers."""
192
- highlights = []
193
- unique_models = set()
194
- for benchmark in benchmarks:
195
- data = benchmark.get("data")
196
- if data is None or "Model" not in getattr(data, "columns", []):
197
- continue
198
- active = data[~data["Model"].astype(str).str.startswith("#")]
199
- unique_models.update(active["Model"].astype(str).tolist())
200
-
201
- score_column = benchmark.get("overall_column")
202
- score_columns = benchmark.get("score_columns") or []
203
- if not score_column or score_column not in data.columns:
204
- score_column = score_columns[0] if score_columns else None
205
- top = _top_models(data, score_column, n=1) if score_column else []
206
- if not top:
207
- continue
208
- model, score = top[0]
209
- highlights.append(
210
- {
211
- "label": f"BEST {benchmark['title'].upper()}",
212
- "model": model,
213
- "detail": f"{_display_label(score_column)} · {_format_score(score)}",
214
- }
215
- )
216
 
217
- if unique_models:
218
- highlights.append(
219
- {
220
- "label": "MODELS SCORED",
221
- "model": str(len(unique_models)),
222
- "detail": "unique across prompt suites",
223
- }
224
- )
225
- return highlights
226
 
 
 
 
 
 
 
 
 
 
 
 
 
227
 
228
- def render_home(benchmarks):
229
- highlights = _home_highlights(benchmarks)
230
 
231
- gr.Markdown(
232
- """
233
- P-Bench ranks **text-to-image models** on quality, speed, and price using
234
- the same prompt suites. Open a benchmark for tables, Pareto graphs, and
235
- sample comparisons.
236
- """,
237
- elem_classes="home-intro",
238
- )
239
 
240
- if highlights:
241
- callout_bits = [
242
- f"<div><span>{escape(item['label'])}</span>"
243
- f"<strong>{escape(item['model'])}</strong>"
244
- f"<em>{escape(item['detail'])}</em></div>"
245
- for item in highlights
246
- ]
247
- gr.HTML(
248
- f'<div class="home-callouts">{"".join(callout_bits)}</div>',
249
- padding=False,
250
- elem_classes="home-callouts-host",
251
- )
252
 
253
- gr.Markdown("## Benchmark snapshots", elem_classes="home-section-title")
254
- cards = []
255
- for benchmark in benchmarks:
256
- data = benchmark["data"]
257
- score_column = benchmark.get("overall_column")
258
- score_columns = benchmark.get("score_columns") or []
259
- if not score_column or score_column not in data.columns:
260
- score_column = score_columns[0] if score_columns else None
261
- top = _top_models(data, score_column, n=3) if score_column else []
262
- score_label = _display_label(score_column) if score_column else "Score"
263
- rows_html = "".join(
264
- f"<li><span class='home-rank'>{idx}</span>"
265
- f"<span class='home-model'>{escape(model)}</span>"
266
- f"<span class='home-score'>{_format_score(score)}</span></li>"
267
- for idx, (model, score) in enumerate(top, start=1)
268
- ) or "<li class='home-empty'>No scores yet.</li>"
269
- cards.append(
270
- "<article class='home-snap-card'>"
271
- "<div class='home-snap-title'>"
272
- f"<span class='home-snap-emoji' aria-hidden='true'>{escape(benchmark.get('emoji', '📊'))}</span>"
273
- f"<span>{escape(benchmark['title'])}</span>"
274
- "</div>"
275
- f"<p class='home-snap-blurb'>{escape(benchmark.get('card_description', ''))}</p>"
276
- "<div class='home-snap-top'>"
277
- f"<div class='home-top-label'>Top 3 by {escape(score_label)}</div>"
278
- f"<ol class='home-top-list'>{rows_html}</ol>"
279
- "</div>"
280
- "</article>"
281
- )
282
- gr.HTML(
283
- f"<div class='home-snapshots-grid'>{''.join(cards)}</div>",
284
- padding=False,
285
- elem_classes="home-snapshots-host",
286
- )
 
 
 
 
 
 
 
 
 
 
287
 
288
 
289
  def _format_leaderboard_cell(column, value):
@@ -390,121 +362,17 @@ def _leaderboard_html(data, columns, score_columns, overall_column):
390
  """
391
 
392
 
393
- def render_leaderboard(
394
- data,
395
- columns,
396
- note=None,
397
- score_columns=None,
398
- overall_column=None,
399
- ):
400
- score_columns = list(score_columns or _infer_score_columns(columns))
401
- overall_column = overall_column or _default_overall_column(score_columns)
402
- platform_choices = _filter_choices(data, "Platform")
403
- owner_choices = _filter_choices(data, "Endpoint Owner")
404
- optimized_choices = _filter_choices(data, "Optimized")
405
-
406
- if note:
407
- gr.Markdown(note)
408
-
409
- filter_inputs = []
410
- with gr.Row(elem_classes="leaderboard-controls"):
411
- search = gr.Textbox(
412
- label="Search models",
413
- placeholder="Type a model or provider name…",
414
- scale=3,
415
- max_lines=1,
416
- elem_classes="leaderboard-search",
417
- )
418
- filter_inputs.append(search)
419
- platform = None
420
- owner = None
421
- optimized = None
422
- if platform_choices:
423
- platform = gr.Dropdown(
424
- choices=platform_choices,
425
- value=[],
426
- label="Providers",
427
- multiselect=True,
428
- scale=1,
429
- )
430
- filter_inputs.append(platform)
431
- if owner_choices:
432
- owner = gr.Dropdown(
433
- choices=owner_choices,
434
- value=[],
435
- label="Endpoint owners",
436
- multiselect=True,
437
- scale=1,
438
- )
439
- filter_inputs.append(owner)
440
- if optimized_choices:
441
- optimized = gr.Dropdown(
442
- choices=optimized_choices,
443
- value=[],
444
- label="Optimized",
445
- multiselect=True,
446
- scale=1,
447
- )
448
- filter_inputs.append(optimized)
449
-
450
- ranking = gr.HTML(
451
- _leaderboard_html(data, columns, score_columns, overall_column),
452
- padding=False,
453
- elem_classes="ranking-table-host",
454
- )
455
-
456
- def update_ranking(
457
- search_term,
458
- platform_value=None,
459
- owner_value=None,
460
- optimized_value=None,
461
- ):
462
- filtered_data = _filter_leaderboard(
463
- data,
464
- search_term,
465
- platform_value or [],
466
- owner_value or [],
467
- optimized_value or [],
468
- )
469
- return _leaderboard_html(
470
- filtered_data, columns, score_columns, overall_column
471
- )
472
-
473
- # Wire only the filters that actually exist for this table.
474
- change_inputs = [search]
475
- if platform is not None:
476
- change_inputs.append(platform)
477
- if owner is not None:
478
- change_inputs.append(owner)
479
- if optimized is not None:
480
- change_inputs.append(optimized)
481
-
482
- for component in filter_inputs:
483
- component.change(
484
- update_ranking,
485
- inputs=change_inputs,
486
- outputs=ranking,
487
- )
488
-
489
-
490
- def _infer_score_columns(columns):
491
- return [column for column in columns if column.startswith("OneIG (")]
492
-
493
-
494
- def _default_overall_column(score_columns):
495
- if len(score_columns) == 1:
496
- return score_columns[0]
497
- return "OneIG Overall Score"
498
-
499
-
500
  def _filter_choices(data, column):
501
- if column not in data.columns:
502
  return []
503
  return sorted(data[column].dropna().astype(str).unique().tolist())
504
 
505
 
506
- def _filter_leaderboard(data, search_term, platform, owner, optimized):
507
  filtered = data.copy()
 
 
 
508
  if search_term:
509
  search_columns = [
510
  column
@@ -528,9 +396,7 @@ def _filter_leaderboard(data, search_term, platform, owner, optimized):
528
  return filtered
529
 
530
 
531
- def _leaderboard_dataframe(data, columns, score_columns, overall_column):
532
- # Honor the caller-provided column list so extra metrics (e.g. Elo) are not
533
- # dropped just because they are not part of the ranking score_columns.
534
  skip_columns = {"URL", "Rank"}
535
  preferred_prefix = [
536
  column
@@ -548,8 +414,6 @@ def _leaderboard_dataframe(data, columns, score_columns, overall_column):
548
  ]
549
  if column in data.columns
550
  ]
551
- # Keep overall_column visible when the caller includes it (e.g. Datapoint Elo).
552
- # Synthetic aggregates like OneIG Overall Score are simply omitted from `columns`.
553
  middle = [
554
  column
555
  for column in columns
@@ -558,6 +422,14 @@ def _leaderboard_dataframe(data, columns, score_columns, overall_column):
558
  and column not in preferred_prefix
559
  and column not in preferred_suffix
560
  ]
 
 
 
 
 
 
 
 
561
 
562
  ordered_columns = []
563
  seen = set()
@@ -568,7 +440,6 @@ def _leaderboard_dataframe(data, columns, score_columns, overall_column):
568
 
569
  leaderboard = data[ordered_columns].copy()
570
 
571
- # Rank by overall when available, even if that column is not displayed.
572
  if overall_column and overall_column in data.columns:
573
  leaderboard = (
574
  leaderboard.assign(_sort_key=data[overall_column])
@@ -596,7 +467,7 @@ def _display_label(column):
596
  "P-Judge Overall": "P-Judger (Pruna)",
597
  "Datapoint Elo": "Datapoint Elo",
598
  "Rapidata Elo": "Rapidata Elo",
599
- "Benchmark.ai Elo": "Benchmark.ai Elo",
600
  "Raw Win Rate": "Raw win rate",
601
  "Median Generation Time (s)": "Median generation time",
602
  "Min Generation Time (s)": "Min generation time",
@@ -607,314 +478,10 @@ def _display_label(column):
607
  return labels.get(column, column)
608
 
609
 
610
- def _format_score(value):
611
- return "-" if pd.isna(value) or value is None else f"{float(value):.3f}"
612
-
613
-
614
  def _format_price(value):
615
  return "-" if pd.isna(value) or value is None else f"${float(value):.3f}"
616
 
617
 
618
- def render_benchmark_detail(benchmark):
619
- gr.Markdown(
620
- f"""
621
- # {benchmark["title"]}
622
-
623
- {benchmark["intro"]}
624
- """
625
- )
626
- view_menu = gr.Radio(
627
- choices=["Leaderboard", "Graphs", "Compare samples"],
628
- value="Leaderboard",
629
- show_label=False,
630
- container=False,
631
- elem_classes="benchmark-view-menu",
632
- )
633
- with gr.Column(visible=True, min_width=0, elem_classes="benchmark-panel") as leaderboard_view:
634
- render_leaderboard(
635
- benchmark["data"],
636
- benchmark["columns"],
637
- note=benchmark.get("note"),
638
- score_columns=benchmark.get("score_columns"),
639
- overall_column=benchmark.get("overall_column"),
640
- )
641
- with gr.Column(visible=False, min_width=0, elem_classes="benchmark-panel") as graphs_view:
642
- render_benchmark_graphs(benchmark)
643
- with gr.Column(visible=False, min_width=0, elem_classes="benchmark-panel") as compare_view:
644
- render_compare_samples(benchmark)
645
-
646
- def switch_view(choice):
647
- return (
648
- gr.update(visible=choice == "Leaderboard"),
649
- gr.update(visible=choice == "Graphs"),
650
- gr.update(visible=choice == "Compare samples"),
651
- )
652
-
653
- view_menu.change(
654
- switch_view,
655
- inputs=view_menu,
656
- outputs=[leaderboard_view, graphs_view, compare_view],
657
- )
658
-
659
-
660
- def render_compare_samples(benchmark):
661
- samples = benchmark.get("samples")
662
- if not samples:
663
- gr.Markdown(
664
- """
665
- Sample comparison is not available for this benchmark yet.
666
-
667
- When generations are linked, you will be able to pick models and browse
668
- side-by-side outputs for the same prompts.
669
- """
670
- )
671
- return
672
-
673
- models = samples["models"]
674
- default_models = models[: min(2, len(models))]
675
-
676
- gr.Markdown(
677
- f"""
678
- <p class="compare-samples-help">
679
- Pick up to <strong>{MAX_COMPARE_MODELS}</strong> models to compare side by
680
- side. Images come from the public generation URLs for this benchmark.
681
- Prompts to show chooses how many shared prompts appear (1–{MAX_COMPARE_PROMPTS}).
682
- </p>
683
- """
684
- )
685
- with gr.Row(equal_height=False, elem_classes="compare-controls"):
686
- model_picker = gr.Dropdown(
687
- choices=models,
688
- value=default_models,
689
- multiselect=True,
690
- max_choices=MAX_COMPARE_MODELS,
691
- label="Models",
692
- container=False,
693
- scale=4,
694
- min_width=220,
695
- elem_classes="compare-models",
696
- )
697
- prompt_count = gr.Slider(
698
- minimum=1,
699
- maximum=MAX_COMPARE_PROMPTS,
700
- value=DEFAULT_COMPARE_PROMPTS,
701
- step=1,
702
- label="Prompts to show",
703
- container=False,
704
- show_reset_button=False,
705
- scale=1,
706
- min_width=180,
707
- elem_classes="compare-prompt-count",
708
- )
709
- shuffle_button = gr.Button(
710
- "Shuffle prompts",
711
- variant="primary",
712
- scale=0,
713
- min_width=140,
714
- elem_classes="compare-shuffle",
715
- )
716
-
717
- gallery = gr.HTML(
718
- value=_build_compare_samples_html(
719
- samples,
720
- default_models,
721
- DEFAULT_COMPARE_PROMPTS,
722
- seed=0,
723
- ),
724
- elem_classes="compare-gallery",
725
- )
726
- seed_state = gr.State(0)
727
-
728
- def update_gallery(selected_models, num_prompts, seed):
729
- return _build_compare_samples_html(
730
- samples,
731
- selected_models,
732
- int(num_prompts),
733
- seed=int(seed or 0),
734
- )
735
-
736
- def shuffle_gallery(selected_models, num_prompts, seed):
737
- next_seed = int(seed or 0) + 1
738
- return next_seed, _build_compare_samples_html(
739
- samples,
740
- selected_models,
741
- int(num_prompts),
742
- seed=next_seed,
743
- )
744
-
745
- model_picker.change(
746
- update_gallery,
747
- inputs=[model_picker, prompt_count, seed_state],
748
- outputs=gallery,
749
- )
750
- prompt_count.change(
751
- update_gallery,
752
- inputs=[model_picker, prompt_count, seed_state],
753
- outputs=gallery,
754
- )
755
- shuffle_button.click(
756
- shuffle_gallery,
757
- inputs=[model_picker, prompt_count, seed_state],
758
- outputs=[seed_state, gallery],
759
- )
760
-
761
-
762
- def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
763
- selected_models = [
764
- model
765
- for model in (selected_models or [])
766
- if model in samples["images"]
767
- ][:MAX_COMPARE_MODELS]
768
-
769
- if not selected_models:
770
- return (
771
- '<div class="compare-empty">'
772
- "Select at least one model to compare samples."
773
- "</div>"
774
- )
775
-
776
- shared_prompt_ids = None
777
- for model in selected_models:
778
- model_prompt_ids = set(samples["images"][model])
779
- shared_prompt_ids = (
780
- model_prompt_ids
781
- if shared_prompt_ids is None
782
- else shared_prompt_ids & model_prompt_ids
783
- )
784
-
785
- shared_prompt_ids = sorted(shared_prompt_ids or [])
786
- if not shared_prompt_ids:
787
- return (
788
- '<div class="compare-empty">'
789
- "No shared prompts found for the selected models."
790
- "</div>"
791
- )
792
-
793
- rng = random.Random(seed)
794
- prompt_pool = list(shared_prompt_ids)
795
- rng.shuffle(prompt_pool)
796
- chosen = prompt_pool[: max(1, min(int(num_prompts), len(prompt_pool)))]
797
-
798
- columns = len(selected_models)
799
- blocks = []
800
- for index, prompt_id in enumerate(chosen, start=1):
801
- prompt_text = escape(samples["prompts"].get(prompt_id, ""))
802
- cells = []
803
- for model in selected_models:
804
- image_url = escape(samples["images"][model][prompt_id], quote=True)
805
- cells.append(
806
- f"""
807
- <div class="compare-cell">
808
- <div class="compare-model-label">{escape(model)}</div>
809
- <a href="{image_url}" target="_blank" rel="noopener noreferrer">
810
- <img src="{image_url}" alt="{escape(model)} sample" loading="lazy" />
811
- </a>
812
- </div>
813
- """
814
- )
815
- blocks.append(
816
- f"""
817
- <div class="compare-prompt-block">
818
- <div class="compare-prompt-meta">
819
- <span>Prompt {index}</span>
820
- <span>{escape(prompt_id)}</span>
821
- </div>
822
- <p class="compare-prompt-text">{prompt_text}</p>
823
- <div class="compare-row" style="grid-template-columns: repeat({columns}, minmax(0, 1fr));">
824
- {''.join(cells)}
825
- </div>
826
- </div>
827
- """
828
- )
829
-
830
- return "\n".join(blocks)
831
-
832
-
833
- def render_benchmarks(benchmarks):
834
- """Catalogue cards + detail pages; back button returns to the catalogue."""
835
- open_buttons = []
836
- detail_entries = []
837
-
838
- with gr.Column(visible=True, elem_classes="benchmark-catalogue") as catalogue:
839
- gr.Markdown(
840
- """
841
- # Benchmarks
842
-
843
- Choose a prompt suite. Each one has a **Leaderboard** table, **Graphs**,
844
- and **Compare samples**.
845
- """
846
- )
847
- with gr.Row(equal_height=True, elem_classes="benchmark-catalogue-row"):
848
- for benchmark in benchmarks:
849
- with gr.Column(
850
- scale=1,
851
- min_width=280,
852
- elem_classes="benchmark-card-col",
853
- ):
854
- gr.HTML(
855
- f"""
856
- <div class="benchmark-card-body">
857
- <div class="benchmark-card-title">
858
- <span class="benchmark-card-emoji" aria-hidden="true">{escape(benchmark.get("emoji", "📊"))}</span>
859
- <span>{escape(benchmark["title"])}</span>
860
- </div>
861
- <p class="benchmark-card-blurb">
862
- {escape(benchmark.get("card_description", ""))}
863
- </p>
864
- </div>
865
- """,
866
- padding=False,
867
- )
868
- open_buttons.append(
869
- (
870
- benchmark["id"],
871
- gr.Button(
872
- "View benchmark →",
873
- variant="primary",
874
- size="sm",
875
- elem_classes="benchmark-open-btn",
876
- ),
877
- )
878
- )
879
-
880
- for benchmark in benchmarks:
881
- with gr.Column(visible=False, min_width=0, elem_classes="benchmark-detail") as detail:
882
- back_button = gr.Button(
883
- "← All benchmarks",
884
- elem_classes="benchmark-back-btn",
885
- )
886
- render_benchmark_detail(benchmark)
887
- detail_entries.append((benchmark["id"], detail, back_button))
888
-
889
- nav_outputs = [catalogue, *[detail for _, detail, _ in detail_entries]]
890
-
891
- def show_catalogue(_evt=None):
892
- return (
893
- gr.Column(visible=True),
894
- *[gr.Column(visible=False) for _ in detail_entries],
895
- )
896
-
897
- def show_detail(selected_id):
898
- return (
899
- gr.Column(visible=False),
900
- *[
901
- gr.Column(visible=(benchmark_id == selected_id))
902
- for benchmark_id, _, _ in detail_entries
903
- ],
904
- )
905
-
906
- for benchmark_id, button in open_buttons:
907
- button.click(
908
- lambda selected_id=benchmark_id: show_detail(selected_id),
909
- outputs=nav_outputs,
910
- )
911
-
912
- for _, _, back_button in detail_entries:
913
- back_button.click(show_catalogue, outputs=nav_outputs)
914
-
915
- return show_catalogue, nav_outputs
916
-
917
-
918
  def _pareto_frontier_mask(x_values, scores):
919
  """True for non-dominated points when maximizing score and minimizing x."""
920
  n = len(x_values)
@@ -931,6 +498,31 @@ def _pareto_frontier_mask(x_values, scores):
931
  return mask
932
 
933
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
934
  def _build_pareto_figure(
935
  data,
936
  score_column,
@@ -1013,7 +605,6 @@ def _build_pareto_figure(
1013
  "bgcolor": "rgba(0,0,0,0)",
1014
  "font": {"color": "#d4d4d4", "size": 12},
1015
  },
1016
- # Match Pruna playground surfaces (#120b1b / #1d1429).
1017
  plot_bgcolor="#1d1429",
1018
  paper_bgcolor="#171021",
1019
  font={"color": "#d4d4d4", "size": 13},
@@ -1039,110 +630,661 @@ def _build_pareto_figure(
1039
  return fig
1040
 
1041
 
1042
- def render_benchmark_graphs(benchmark):
1043
- data = benchmark["data"]
1044
- score_columns = [
1045
- column
1046
- for column in (benchmark.get("score_columns") or [])
1047
- if column in data.columns
1048
- ]
1049
- overall_column = benchmark.get("overall_column")
1050
-
1051
- if not score_columns and overall_column and overall_column in data.columns:
1052
- score_columns = [overall_column]
1053
-
1054
- if not score_columns:
1055
- gr.Markdown("No score data is available yet.")
1056
- return
1057
-
1058
- # Pareto every displayed quality metric vs price.
1059
- # Skip only synthetic aggregates (e.g. OneIG mean), not real sort keys like Datapoint Elo.
1060
- pareto_skip = {
1061
- "Model",
1062
- "Platform",
1063
- "Endpoint Owner",
1064
- "Optimized",
1065
- "URL",
1066
- "Rank",
1067
- "Median Generation Time (s)",
1068
- "Min Generation Time (s)",
1069
- "Price / Image (USD)",
1070
- "Evaluation Date (UTC)",
1071
- "Date",
1072
- "Raw Win Rate",
1073
- "OneIG Overall Score",
1074
- }
1075
-
1076
- display_columns = benchmark.get("columns") or []
1077
- pareto_columns = []
1078
- for column in [*score_columns, *display_columns]:
1079
- if (
1080
- column in data.columns
1081
- and column not in pareto_skip
1082
- and column not in pareto_columns
1083
- and pd.api.types.is_numeric_dtype(data[column])
1084
- ):
1085
- pareto_columns.append(column)
1086
 
1087
  price_column = "Price / Image (USD)"
1088
  time_column = "Min Generation Time (s)"
1089
- price_figures = []
1090
- time_figures = []
1091
- for plot_column in pareto_columns:
1092
- if price_column in data.columns:
1093
- price_fig = _build_pareto_figure(
1094
- data,
1095
- plot_column,
1096
- x_column=price_column,
1097
- x_title="Price per image (USD)",
1098
- x_hover_prefix="$",
1099
- )
1100
- if price_fig is not None:
1101
- price_figures.append((plot_column, price_fig))
1102
- if time_column in data.columns:
1103
- time_fig = _build_pareto_figure(
1104
- data,
1105
- plot_column,
1106
- x_column=time_column,
1107
- x_title="Min generation time (s)",
1108
- x_hover_suffix="s",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1109
  )
1110
- if time_fig is not None:
1111
- time_figures.append((plot_column, time_fig))
1112
-
1113
- if price_figures or time_figures:
1114
- gr.Markdown(
1115
- "### Pareto frontiers\n\n"
1116
- "<span class='pareto-help'>"
1117
- "Green = on the frontier (lower cost or time at the same or better score). "
1118
- "Lavender = below the frontier."
1119
- "</span><br/>"
1120
- "<strong class='pareto-help-emphasis'>Hover a point to see which model it is.</strong>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1121
  )
1122
- with gr.Row(equal_height=False, elem_classes="pareto-layout"):
1123
- with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
1124
- gr.Markdown("#### Price vs score")
1125
- if not price_figures:
1126
- gr.Markdown("_No price data available._")
1127
- for plot_column, pareto_fig in price_figures:
1128
- gr.Markdown(f"**{_display_label(plot_column)}**")
1129
- gr.Plot(
1130
- value=pareto_fig,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1131
  show_label=False,
1132
  elem_classes="pareto-plot",
1133
  )
1134
- with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
1135
- gr.Markdown("#### Min generation time vs score")
1136
- if not time_figures:
1137
- gr.Markdown("_No min generation time data available._")
1138
- for plot_column, pareto_fig in time_figures:
1139
- gr.Markdown(f"**{_display_label(plot_column)}**")
1140
- gr.Plot(
1141
- value=pareto_fig,
1142
  show_label=False,
1143
  elem_classes="pareto-plot",
1144
  )
1145
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1146
 
1147
  def render_about():
1148
  with gr.Row(elem_classes="about-layout", equal_height=False):
 
24
  # About P-Bench
25
 
26
  P-Bench compares **text-to-image models**, including optimized or accelerated
27
+ endpoints, on **quality, speed, and price**. Each view is a **dataset** scored
28
+ with a **metric**, written as `Dataset | Metric`. There is no single score
29
+ across P-Bench.
30
 
31
  ## How to read it
32
 
33
+ 1. Pick a **dataset** and a **metric**. The title is always Dataset | Metric.
34
+ 2. **Leaderboards**: ranked by that metric. Price and generation time sit in
35
+ the same table.
36
+ 3. **Pareto plots**: mark models that are not beaten on both higher score
37
  and lower price (or time).
38
+ 4. **Samples**: the same prompts, side by side.
 
39
 
40
  ## How a score is made
41
 
 
43
  2. It generates one image per prompt when the run succeeds. Not every model
44
  has every prompt or every metric.
45
  3. Quality is scored automatically (OneIG alignment, P-Judger) and, where
46
+ available, by human preference (Datapoint Elo, Rapidata Elo) or by
47
+ Artificial Analysis Elo.
48
  4. Price per image and generation time are joined from the evaluation table.
49
 
50
+ ## Current datasets
51
 
52
+ ### Qwen Image Dataset
53
+ 100 prompts from the 1,000-prompt Qwen Image Bench set, sampled for coverage
54
+ across its fine-grained (L3) categories. Metrics include Datapoint Elo,
55
+ Rapidata Elo, and P-Judger.
56
+
57
+ ### OneIG Alignment Dataset
58
  Prompt-image **alignment** on anime / stylization, human / portrait, and
59
  general object prompts (100 prompts each). This is the alignment slice of
60
+ OneIG, not the full suite. Alignment Overall is the mean of the category
61
+ scores that exist for that row.
62
 
63
+ ### Artificial Analysis Dataset
64
+ Artificial Analysis Elo from the evaluation table. Rapidata is a **metric**,
65
+ not a dataset.
66
  """
67
 
68
  ABOUT_DETAILS_CONTENT = """
 
78
  - **Datapoint Elo**: human-preference Elo from Datapoint pairwise comparisons.
79
  - **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.
80
  Rapidata rejects prompts over 400 characters, so this Elo is on a subset
81
+ of each suite (see Setup). Rapidata is not a dataset.
82
+ - **Artificial Analysis Elo**: preference Elo from Artificial Analysis, shown
83
+ as its own dataset.
84
  - **Generation time**: median and minimum generation time in seconds, as
85
  reported in the evaluation table. This is not a p95, and we do not state
86
  warm vs cold or concurrent load.
87
  - **Price**: USD per image in the evaluation table. We do not state list
88
  price vs amount paid, or whether failed generations are included.
89
 
90
+ Scores from different datasets or metrics are **not interchangeable**. A high
91
  OneIG alignment score is not the same quantity as a Datapoint Elo. Compare
92
+ models *within* a Dataset | Metric view.
93
 
94
  ## Setup
95
 
 
97
  - **Update policy:** numbers come from evaluation snapshots in the tables,
98
  not a live API poll.
99
  - **Prompt counts:** OneIG Alignment uses the first 100 prompts from each of
100
+ the three categories (300 total). Qwen Image Dataset uses 100 prompts sampled
101
  from the 1,000-prompt pool for roughly even coverage of its fine-grained
102
  (L3) categories.
103
  - **Generation:** one image per prompt per endpoint when the run exists.
 
109
  - **Datapoint:** every model pair is compared on every prompt, with 10 votes
110
  per battle.
111
  - **Rapidata:** prompts longer than 400 characters are dropped, leaving 212
112
+ OneIG prompts and 85 Qwen Image Dataset prompts. 4 votes per pair; about
113
+ 26,000 votes on OneIG and 35,000 on Qwen Image Dataset.
114
 
115
  ## Limits
116
 
117
  - Empty cells mean that track was not run or not reported for that model.
118
  - Rapidata Elo is not on the full prompt suite, so it is not directly
119
+ comparable to Datapoint Elo even on the same dataset.
120
  - Elo ratings can shift when the comparison pool changes: treat them as
121
  relative rankings for the snapshot, not absolute constants.
122
  - Close scores can be a tie in practice; the table does not show confidence
 
179
  )
180
 
181
 
182
+ def _item(items, item_id):
183
+ for item in items:
184
+ if item["id"] == item_id:
185
+ return item
186
+ return items[0] if items else None
 
 
 
 
 
 
 
 
 
187
 
188
 
189
+ def _dataset_choices(datasets):
190
+ return [(dataset["name"], dataset["id"]) for dataset in datasets]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
191
 
 
 
 
 
 
 
 
 
 
192
 
193
+ def _metric_choices(datasets, metrics, dataset_id):
194
+ dataset = _item(datasets, dataset_id)
195
+ if not dataset:
196
+ return []
197
+ allowed = set(dataset.get("metric_ids") or [])
198
+ data = dataset.get("data")
199
+ columns = getattr(data, "columns", [])
200
+ return [
201
+ (metric["name"], metric["id"])
202
+ for metric in metrics
203
+ if metric["id"] in allowed and metric["column"] in columns
204
+ ]
205
 
 
 
206
 
207
+ def _coerce_metric(datasets, metrics, dataset_id, metric_id):
208
+ choices = _metric_choices(datasets, metrics, dataset_id)
209
+ ids = [choice[1] for choice in choices]
210
+ if metric_id in ids:
211
+ return metric_id
212
+ return ids[0] if ids else None
 
 
213
 
 
 
 
 
 
 
 
 
 
 
 
 
214
 
215
+ def _model_choices(datasets, dataset_id):
216
+ dataset = _item(datasets, dataset_id)
217
+ data = dataset.get("data") if dataset else None
218
+ if data is None or "Model" not in getattr(data, "columns", []):
219
+ return []
220
+ return sorted(data["Model"].dropna().astype(str).unique().tolist())
221
+
222
+
223
+ def _view_title(datasets, metrics, dataset_id, metric_id):
224
+ dataset = _item(datasets, dataset_id)
225
+ metric = _item(metrics, metric_id)
226
+ dataset_name = dataset["name"] if dataset else "Dataset"
227
+ metric_name = metric["name"] if metric else "Metric"
228
+ return f"{dataset_name} | {metric_name}"
229
+
230
+
231
+ def _columns_for_metric(dataset, metric_column):
232
+ columns = list(dataset.get("columns") or [])
233
+ if metric_column and metric_column not in columns:
234
+ identity = {"Model", "Platform", "Endpoint Owner", "Optimized"}
235
+ insert_at = 0
236
+ for index, column in enumerate(columns):
237
+ if column in identity:
238
+ insert_at = index + 1
239
+ columns.insert(insert_at, metric_column)
240
+ return columns
241
+
242
+
243
+ def resolve_view(datasets, metrics, dataset_id, metric_id):
244
+ dataset = _item(datasets, dataset_id)
245
+ metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
246
+ metric = _item(metrics, metric_id)
247
+ if not dataset or not metric:
248
+ return None
249
+ return {
250
+ "dataset": dataset,
251
+ "metric": metric,
252
+ "title": _view_title(datasets, metrics, dataset["id"], metric["id"]),
253
+ "data": dataset["data"],
254
+ "columns": _columns_for_metric(dataset, metric["column"]),
255
+ "score_column": metric["column"],
256
+ "samples": dataset.get("samples"),
257
+ "note": dataset.get("note"),
258
+ }
259
 
260
 
261
  def _format_leaderboard_cell(column, value):
 
362
  """
363
 
364
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
365
  def _filter_choices(data, column):
366
+ if data is None or column not in data.columns:
367
  return []
368
  return sorted(data[column].dropna().astype(str).unique().tolist())
369
 
370
 
371
+ def _filter_leaderboard(data, search_term, platform, owner, optimized, models=None):
372
  filtered = data.copy()
373
+ if models:
374
+ if "Model" in filtered.columns:
375
+ filtered = filtered[filtered["Model"].astype(str).isin(models)]
376
  if search_term:
377
  search_columns = [
378
  column
 
396
  return filtered
397
 
398
 
399
+ def _leaderboard_dataframe(data, columns, score_columns, overall_column): # noqa: ARG001
 
 
400
  skip_columns = {"URL", "Rank"}
401
  preferred_prefix = [
402
  column
 
414
  ]
415
  if column in data.columns
416
  ]
 
 
417
  middle = [
418
  column
419
  for column in columns
 
422
  and column not in preferred_prefix
423
  and column not in preferred_suffix
424
  ]
425
+ if (
426
+ overall_column
427
+ and overall_column in data.columns
428
+ and overall_column not in middle
429
+ and overall_column not in preferred_prefix
430
+ and overall_column not in preferred_suffix
431
+ ):
432
+ middle.insert(0, overall_column)
433
 
434
  ordered_columns = []
435
  seen = set()
 
440
 
441
  leaderboard = data[ordered_columns].copy()
442
 
 
443
  if overall_column and overall_column in data.columns:
444
  leaderboard = (
445
  leaderboard.assign(_sort_key=data[overall_column])
 
467
  "P-Judge Overall": "P-Judger (Pruna)",
468
  "Datapoint Elo": "Datapoint Elo",
469
  "Rapidata Elo": "Rapidata Elo",
470
+ "Benchmark.ai Elo": "Artificial Analysis Elo",
471
  "Raw Win Rate": "Raw win rate",
472
  "Median Generation Time (s)": "Median generation time",
473
  "Min Generation Time (s)": "Min generation time",
 
478
  return labels.get(column, column)
479
 
480
 
 
 
 
 
481
  def _format_price(value):
482
  return "-" if pd.isna(value) or value is None else f"${float(value):.3f}"
483
 
484
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
485
  def _pareto_frontier_mask(x_values, scores):
486
  """True for non-dominated points when maximizing score and minimizing x."""
487
  n = len(x_values)
 
498
  return mask
499
 
500
 
501
+ def _empty_figure(message):
502
+ fig = go.Figure()
503
+ fig.add_annotation(
504
+ text=message,
505
+ xref="paper",
506
+ yref="paper",
507
+ x=0.5,
508
+ y=0.5,
509
+ showarrow=False,
510
+ font={"color": "#a3a3a3", "size": 14},
511
+ )
512
+ fig.update_layout(
513
+ title=None,
514
+ autosize=True,
515
+ height=420,
516
+ margin={"l": 56, "r": 28, "t": 28, "b": 80},
517
+ plot_bgcolor="#1d1429",
518
+ paper_bgcolor="#171021",
519
+ font={"color": "#d4d4d4", "size": 13},
520
+ xaxis={"visible": False},
521
+ yaxis={"visible": False},
522
+ )
523
+ return fig
524
+
525
+
526
  def _build_pareto_figure(
527
  data,
528
  score_column,
 
605
  "bgcolor": "rgba(0,0,0,0)",
606
  "font": {"color": "#d4d4d4", "size": 12},
607
  },
 
608
  plot_bgcolor="#1d1429",
609
  paper_bgcolor="#171021",
610
  font={"color": "#d4d4d4", "size": 13},
 
630
  return fig
631
 
632
 
633
+ def _pareto_pair(data, score_column):
634
+ if data is None or not score_column or score_column not in data.columns:
635
+ empty = _empty_figure("No score data is available yet.")
636
+ return empty, empty
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
637
 
638
  price_column = "Price / Image (USD)"
639
  time_column = "Min Generation Time (s)"
640
+ price_fig = None
641
+ time_fig = None
642
+ if price_column in data.columns:
643
+ price_fig = _build_pareto_figure(
644
+ data,
645
+ score_column,
646
+ x_column=price_column,
647
+ x_title="Price per image (USD)",
648
+ x_hover_prefix="$",
649
+ )
650
+ if time_column in data.columns:
651
+ time_fig = _build_pareto_figure(
652
+ data,
653
+ score_column,
654
+ x_column=time_column,
655
+ x_title="Min generation time (s)",
656
+ x_hover_suffix="s",
657
+ )
658
+ return (
659
+ price_fig or _empty_figure("No price data available."),
660
+ time_fig or _empty_figure("No min generation time data available."),
661
+ )
662
+
663
+
664
+ def _samples_html(samples, selected_models, num_prompts, seed=0):
665
+ if not samples:
666
+ return (
667
+ '<div class="compare-empty">'
668
+ "Sample comparison is not available for this dataset yet."
669
+ "</div>"
670
+ )
671
+ models = list(selected_models or [])
672
+ available = samples.get("models") or []
673
+ models = [model for model in models if model in samples.get("images", {})]
674
+ if not models:
675
+ models = available[: min(2, len(available))]
676
+ return _build_compare_samples_html(samples, models, num_prompts, seed)
677
+
678
+
679
+ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
680
+ selected_models = [
681
+ model
682
+ for model in (selected_models or [])
683
+ if model in samples["images"]
684
+ ][:MAX_COMPARE_MODELS]
685
+
686
+ if not selected_models:
687
+ return (
688
+ '<div class="compare-empty">'
689
+ "Select at least one model to compare samples."
690
+ "</div>"
691
+ )
692
+
693
+ shared_prompt_ids = None
694
+ for model in selected_models:
695
+ model_prompt_ids = set(samples["images"][model])
696
+ shared_prompt_ids = (
697
+ model_prompt_ids
698
+ if shared_prompt_ids is None
699
+ else shared_prompt_ids & model_prompt_ids
700
+ )
701
+
702
+ shared_prompt_ids = sorted(shared_prompt_ids or [])
703
+ if not shared_prompt_ids:
704
+ return (
705
+ '<div class="compare-empty">'
706
+ "No shared prompts found for the selected models."
707
+ "</div>"
708
+ )
709
+
710
+ rng = random.Random(seed)
711
+ prompt_pool = list(shared_prompt_ids)
712
+ rng.shuffle(prompt_pool)
713
+ chosen = prompt_pool[: max(1, min(int(num_prompts), len(prompt_pool)))]
714
+
715
+ columns = len(selected_models)
716
+ blocks = []
717
+ for index, prompt_id in enumerate(chosen, start=1):
718
+ prompt_text = escape(samples["prompts"].get(prompt_id, ""))
719
+ cells = []
720
+ for model in selected_models:
721
+ image_url = escape(samples["images"][model][prompt_id], quote=True)
722
+ cells.append(
723
+ f"""
724
+ <div class="compare-cell">
725
+ <div class="compare-model-label">{escape(model)}</div>
726
+ <a href="{image_url}" target="_blank" rel="noopener noreferrer">
727
+ <img src="{image_url}" alt="{escape(model)} sample" loading="lazy" />
728
+ </a>
729
+ </div>
730
+ """
731
  )
732
+ blocks.append(
733
+ f"""
734
+ <div class="compare-prompt-block">
735
+ <div class="compare-prompt-meta">
736
+ <span>Prompt {index}</span>
737
+ <span>{escape(prompt_id)}</span>
738
+ </div>
739
+ <p class="compare-prompt-text">{prompt_text}</p>
740
+ <div class="compare-row" style="grid-template-columns: repeat({columns}, minmax(0, 1fr));">
741
+ {''.join(cells)}
742
+ </div>
743
+ </div>
744
+ """
745
+ )
746
+
747
+ return "\n".join(blocks)
748
+
749
+
750
+ def _title_markdown(title):
751
+ return f"# {title}"
752
+
753
+
754
+ def _note_markdown(note):
755
+ return note or ""
756
+
757
+
758
+ def _filter_row(datasets, metrics, default_dataset_id, default_metric_id):
759
+ metric_choices = _metric_choices(datasets, metrics, default_dataset_id)
760
+ model_choices = _model_choices(datasets, default_dataset_id)
761
+ with gr.Row(elem_classes="view-filters"):
762
+ dataset_dd = gr.Dropdown(
763
+ choices=_dataset_choices(datasets),
764
+ value=default_dataset_id,
765
+ label="Dataset",
766
+ type="value",
767
+ scale=2,
768
+ min_width=160,
769
+ )
770
+ metric_dd = gr.Dropdown(
771
+ choices=metric_choices,
772
+ value=default_metric_id,
773
+ label="Metric",
774
+ type="value",
775
+ scale=2,
776
+ min_width=180,
777
  )
778
+ models_dd = gr.Dropdown(
779
+ choices=model_choices,
780
+ value=[],
781
+ multiselect=True,
782
+ label="Models",
783
+ type="value",
784
+ scale=3,
785
+ min_width=200,
786
+ )
787
+ title = gr.Markdown(
788
+ _title_markdown(
789
+ _view_title(datasets, metrics, default_dataset_id, default_metric_id)
790
+ ),
791
+ elem_classes="view-title",
792
+ )
793
+ return dataset_dd, metric_dd, models_dd, title
794
+
795
+
796
+ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric_id):
797
+ default_metric_id = _coerce_metric(
798
+ datasets, metrics, default_dataset_id, default_metric_id
799
+ )
800
+ initial = resolve_view(datasets, metrics, default_dataset_id, default_metric_id)
801
+ initial_data = initial["data"]
802
+ initial_columns = initial["columns"]
803
+ initial_score = initial["score_column"]
804
+ initial_samples = initial.get("samples")
805
+ price_fig, time_fig = _pareto_pair(initial_data, initial_score)
806
+
807
+ with gr.Tabs(elem_classes="main-tabs"):
808
+ with gr.TabItem("Leaderboards"):
809
+ lb_dataset, lb_metric, lb_models, lb_title = _filter_row(
810
+ datasets, metrics, default_dataset_id, default_metric_id
811
+ )
812
+ lb_note = gr.Markdown(_note_markdown(initial.get("note")))
813
+ platform_choices = _filter_choices(initial_data, "Platform")
814
+ owner_choices = _filter_choices(initial_data, "Endpoint Owner")
815
+ optimized_choices = _filter_choices(initial_data, "Optimized")
816
+ with gr.Row(elem_classes="leaderboard-controls"):
817
+ search = gr.Textbox(
818
+ label="Search models",
819
+ placeholder="Type a model or provider name…",
820
+ scale=3,
821
+ max_lines=1,
822
+ elem_classes="leaderboard-search",
823
+ )
824
+ platform = gr.Dropdown(
825
+ choices=platform_choices,
826
+ value=[],
827
+ label="Providers",
828
+ multiselect=True,
829
+ scale=1,
830
+ visible=bool(platform_choices),
831
+ )
832
+ owner = gr.Dropdown(
833
+ choices=owner_choices,
834
+ value=[],
835
+ label="Endpoint owners",
836
+ multiselect=True,
837
+ scale=1,
838
+ visible=bool(owner_choices),
839
+ )
840
+ optimized = gr.Dropdown(
841
+ choices=optimized_choices,
842
+ value=[],
843
+ label="Optimized",
844
+ multiselect=True,
845
+ scale=1,
846
+ visible=bool(optimized_choices),
847
+ )
848
+ ranking = gr.HTML(
849
+ _leaderboard_html(
850
+ initial_data, initial_columns, [initial_score], initial_score
851
+ ),
852
+ padding=False,
853
+ elem_classes="ranking-table-host",
854
+ )
855
+
856
+ with gr.TabItem("Pareto Plots"):
857
+ pp_dataset, pp_metric, pp_models, pp_title = _filter_row(
858
+ datasets, metrics, default_dataset_id, default_metric_id
859
+ )
860
+ gr.Markdown(
861
+ "<span class='pareto-help'>"
862
+ "Green = on the frontier (lower cost or time at the same or better score). "
863
+ "Lavender = below the frontier."
864
+ "</span><br/>"
865
+ "<strong class='pareto-help-emphasis'>Hover a point to see which model it is.</strong>"
866
+ )
867
+ with gr.Row(equal_height=False, elem_classes="pareto-layout"):
868
+ with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
869
+ gr.Markdown("#### Price vs score")
870
+ price_plot = gr.Plot(
871
+ value=price_fig,
872
  show_label=False,
873
  elem_classes="pareto-plot",
874
  )
875
+ with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
876
+ gr.Markdown("#### Min generation time vs score")
877
+ time_plot = gr.Plot(
878
+ value=time_fig,
 
 
 
 
879
  show_label=False,
880
  elem_classes="pareto-plot",
881
  )
882
 
883
+ with gr.TabItem("Samples"):
884
+ sm_dataset, sm_metric, sm_models, sm_title = _filter_row(
885
+ datasets, metrics, default_dataset_id, default_metric_id
886
+ )
887
+ gr.Markdown(
888
+ f"""
889
+ <p class="compare-samples-help">
890
+ Filter models above (up to <strong>{MAX_COMPARE_MODELS}</strong> are
891
+ shown). If none are selected, two defaults appear. Images come from
892
+ the public generation URLs for this dataset. Prompts to show chooses
893
+ how many shared prompts appear (1–{MAX_COMPARE_PROMPTS}).
894
+ </p>
895
+ """
896
+ )
897
+ with gr.Row(equal_height=False, elem_classes="compare-controls"):
898
+ prompt_count = gr.Slider(
899
+ minimum=1,
900
+ maximum=MAX_COMPARE_PROMPTS,
901
+ value=DEFAULT_COMPARE_PROMPTS,
902
+ step=1,
903
+ label="Prompts to show",
904
+ container=False,
905
+ show_reset_button=False,
906
+ scale=1,
907
+ min_width=180,
908
+ elem_classes="compare-prompt-count",
909
+ )
910
+ shuffle_button = gr.Button(
911
+ "Shuffle prompts",
912
+ variant="primary",
913
+ scale=0,
914
+ min_width=140,
915
+ elem_classes="compare-shuffle",
916
+ )
917
+ gallery = gr.HTML(
918
+ value=_samples_html(
919
+ initial_samples, [], DEFAULT_COMPARE_PROMPTS, seed=0
920
+ ),
921
+ elem_classes="compare-gallery",
922
+ )
923
+ seed_state = gr.State(0)
924
+
925
+ with gr.TabItem("About"):
926
+ render_about()
927
+
928
+ def _synced_filters(dataset_id, metric_id, models):
929
+ metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
930
+ model_choices = _model_choices(datasets, dataset_id)
931
+ models = [model for model in (models or []) if model in model_choices]
932
+ metric_choices = _metric_choices(datasets, metrics, dataset_id)
933
+ title = _title_markdown(
934
+ _view_title(datasets, metrics, dataset_id, metric_id)
935
+ )
936
+ dataset_update = gr.update(value=dataset_id)
937
+ metric_update = gr.update(choices=metric_choices, value=metric_id)
938
+ models_update = gr.update(choices=model_choices, value=models)
939
+ return (
940
+ dataset_id,
941
+ metric_id,
942
+ models,
943
+ dataset_update,
944
+ dataset_update,
945
+ dataset_update,
946
+ metric_update,
947
+ metric_update,
948
+ metric_update,
949
+ models_update,
950
+ models_update,
951
+ models_update,
952
+ title,
953
+ title,
954
+ title,
955
+ )
956
+
957
+ def _leaderboard_extras(data, platform_value, owner_value, optimized_value):
958
+ platform_choices = _filter_choices(data, "Platform")
959
+ owner_choices = _filter_choices(data, "Endpoint Owner")
960
+ optimized_choices = _filter_choices(data, "Optimized")
961
+ platform_value = [
962
+ value for value in (platform_value or []) if value in platform_choices
963
+ ]
964
+ owner_value = [
965
+ value for value in (owner_value or []) if value in owner_choices
966
+ ]
967
+ optimized_value = [
968
+ value for value in (optimized_value or []) if value in optimized_choices
969
+ ]
970
+ return (
971
+ gr.update(
972
+ choices=platform_choices,
973
+ value=platform_value,
974
+ visible=bool(platform_choices),
975
+ ),
976
+ gr.update(
977
+ choices=owner_choices,
978
+ value=owner_value,
979
+ visible=bool(owner_choices),
980
+ ),
981
+ gr.update(
982
+ choices=optimized_choices,
983
+ value=optimized_value,
984
+ visible=bool(optimized_choices),
985
+ ),
986
+ platform_value,
987
+ owner_value,
988
+ optimized_value,
989
+ )
990
+
991
+ def _views(
992
+ dataset_id,
993
+ metric_id,
994
+ models,
995
+ search_term,
996
+ platform_value,
997
+ owner_value,
998
+ optimized_value,
999
+ num_prompts,
1000
+ seed,
1001
+ ):
1002
+ view = resolve_view(datasets, metrics, dataset_id, metric_id)
1003
+ data = view["data"]
1004
+ filtered = _filter_leaderboard(
1005
+ data,
1006
+ search_term,
1007
+ platform_value or [],
1008
+ owner_value or [],
1009
+ optimized_value or [],
1010
+ models=models,
1011
+ )
1012
+ ranking_html = _leaderboard_html(
1013
+ filtered,
1014
+ view["columns"],
1015
+ [view["score_column"]],
1016
+ view["score_column"],
1017
+ )
1018
+ pareto_data = _filter_leaderboard(
1019
+ data, "", [], [], [], models=models
1020
+ )
1021
+ next_price, next_time = _pareto_pair(pareto_data, view["score_column"])
1022
+ samples_html = _samples_html(
1023
+ view.get("samples"),
1024
+ models,
1025
+ int(num_prompts or DEFAULT_COMPARE_PROMPTS),
1026
+ int(seed or 0),
1027
+ )
1028
+ return (
1029
+ _note_markdown(view.get("note")),
1030
+ ranking_html,
1031
+ next_price,
1032
+ next_time,
1033
+ samples_html,
1034
+ )
1035
+
1036
+ def on_dataset(
1037
+ dataset_id,
1038
+ metric_id,
1039
+ models,
1040
+ search_term,
1041
+ platform_value,
1042
+ owner_value,
1043
+ optimized_value,
1044
+ num_prompts,
1045
+ seed,
1046
+ ):
1047
+ synced = _synced_filters(dataset_id, metric_id, models)
1048
+ dataset_id, metric_id, models = synced[:3]
1049
+ view = resolve_view(datasets, metrics, dataset_id, metric_id)
1050
+ extras = _leaderboard_extras(
1051
+ view["data"], platform_value, owner_value, optimized_value
1052
+ )
1053
+ views = _views(
1054
+ dataset_id,
1055
+ metric_id,
1056
+ models,
1057
+ search_term,
1058
+ extras[3],
1059
+ extras[4],
1060
+ extras[5],
1061
+ num_prompts,
1062
+ seed,
1063
+ )
1064
+ return (*synced[3:], extras[0], extras[1], extras[2], *views)
1065
+
1066
+ def on_metric(
1067
+ dataset_id,
1068
+ metric_id,
1069
+ models,
1070
+ search_term,
1071
+ platform_value,
1072
+ owner_value,
1073
+ optimized_value,
1074
+ num_prompts,
1075
+ seed,
1076
+ ):
1077
+ metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1078
+ title = _title_markdown(
1079
+ _view_title(datasets, metrics, dataset_id, metric_id)
1080
+ )
1081
+ metric_update = gr.update(value=metric_id)
1082
+ views = _views(
1083
+ dataset_id,
1084
+ metric_id,
1085
+ models,
1086
+ search_term,
1087
+ platform_value,
1088
+ owner_value,
1089
+ optimized_value,
1090
+ num_prompts,
1091
+ seed,
1092
+ )
1093
+ return (metric_update, metric_update, metric_update, title, title, title, *views)
1094
+
1095
+ def on_models(
1096
+ dataset_id,
1097
+ metric_id,
1098
+ models,
1099
+ search_term,
1100
+ platform_value,
1101
+ owner_value,
1102
+ optimized_value,
1103
+ num_prompts,
1104
+ seed,
1105
+ ):
1106
+ model_choices = _model_choices(datasets, dataset_id)
1107
+ models = [model for model in (models or []) if model in model_choices]
1108
+ models_update = gr.update(value=models)
1109
+ views = _views(
1110
+ dataset_id,
1111
+ metric_id,
1112
+ models,
1113
+ search_term,
1114
+ platform_value,
1115
+ owner_value,
1116
+ optimized_value,
1117
+ num_prompts,
1118
+ seed,
1119
+ )
1120
+ return (models_update, models_update, models_update, *views)
1121
+
1122
+ def on_leaderboard_filters(
1123
+ dataset_id,
1124
+ metric_id,
1125
+ models,
1126
+ search_term,
1127
+ platform_value,
1128
+ owner_value,
1129
+ optimized_value,
1130
+ ):
1131
+ view = resolve_view(datasets, metrics, dataset_id, metric_id)
1132
+ filtered = _filter_leaderboard(
1133
+ view["data"],
1134
+ search_term,
1135
+ platform_value or [],
1136
+ owner_value or [],
1137
+ optimized_value or [],
1138
+ models=models,
1139
+ )
1140
+ return _leaderboard_html(
1141
+ filtered,
1142
+ view["columns"],
1143
+ [view["score_column"]],
1144
+ view["score_column"],
1145
+ )
1146
+
1147
+ def on_samples_controls(dataset_id, models, num_prompts, seed):
1148
+ view = resolve_view(datasets, metrics, dataset_id, None)
1149
+ return _samples_html(
1150
+ view.get("samples") if view else None,
1151
+ models,
1152
+ int(num_prompts or DEFAULT_COMPARE_PROMPTS),
1153
+ int(seed or 0),
1154
+ )
1155
+
1156
+ def on_shuffle(dataset_id, models, num_prompts, seed):
1157
+ next_seed = int(seed or 0) + 1
1158
+ view = resolve_view(datasets, metrics, dataset_id, None)
1159
+ return next_seed, _samples_html(
1160
+ view.get("samples") if view else None,
1161
+ models,
1162
+ int(num_prompts or DEFAULT_COMPARE_PROMPTS),
1163
+ next_seed,
1164
+ )
1165
+
1166
+ dataset_inputs = [
1167
+ lb_metric,
1168
+ lb_models,
1169
+ search,
1170
+ platform,
1171
+ owner,
1172
+ optimized,
1173
+ prompt_count,
1174
+ seed_state,
1175
+ ]
1176
+ dataset_outputs = [
1177
+ lb_dataset,
1178
+ pp_dataset,
1179
+ sm_dataset,
1180
+ lb_metric,
1181
+ pp_metric,
1182
+ sm_metric,
1183
+ lb_models,
1184
+ pp_models,
1185
+ sm_models,
1186
+ lb_title,
1187
+ pp_title,
1188
+ sm_title,
1189
+ platform,
1190
+ owner,
1191
+ optimized,
1192
+ lb_note,
1193
+ ranking,
1194
+ price_plot,
1195
+ time_plot,
1196
+ gallery,
1197
+ ]
1198
+ for dataset_dd in (lb_dataset, pp_dataset, sm_dataset):
1199
+ dataset_dd.change(
1200
+ on_dataset,
1201
+ inputs=[dataset_dd, *dataset_inputs],
1202
+ outputs=dataset_outputs,
1203
+ )
1204
+
1205
+ metric_outputs = [
1206
+ lb_metric,
1207
+ pp_metric,
1208
+ sm_metric,
1209
+ lb_title,
1210
+ pp_title,
1211
+ sm_title,
1212
+ lb_note,
1213
+ ranking,
1214
+ price_plot,
1215
+ time_plot,
1216
+ gallery,
1217
+ ]
1218
+ for metric_dd in (lb_metric, pp_metric, sm_metric):
1219
+ metric_dd.change(
1220
+ on_metric,
1221
+ inputs=[
1222
+ lb_dataset,
1223
+ metric_dd,
1224
+ lb_models,
1225
+ search,
1226
+ platform,
1227
+ owner,
1228
+ optimized,
1229
+ prompt_count,
1230
+ seed_state,
1231
+ ],
1232
+ outputs=metric_outputs,
1233
+ )
1234
+
1235
+ models_outputs = [
1236
+ lb_models,
1237
+ pp_models,
1238
+ sm_models,
1239
+ lb_note,
1240
+ ranking,
1241
+ price_plot,
1242
+ time_plot,
1243
+ gallery,
1244
+ ]
1245
+ for models_dd in (lb_models, pp_models, sm_models):
1246
+ models_dd.change(
1247
+ on_models,
1248
+ inputs=[
1249
+ lb_dataset,
1250
+ lb_metric,
1251
+ models_dd,
1252
+ search,
1253
+ platform,
1254
+ owner,
1255
+ optimized,
1256
+ prompt_count,
1257
+ seed_state,
1258
+ ],
1259
+ outputs=models_outputs,
1260
+ )
1261
+
1262
+ for component in (search, platform, owner, optimized):
1263
+ component.change(
1264
+ on_leaderboard_filters,
1265
+ inputs=[
1266
+ lb_dataset,
1267
+ lb_metric,
1268
+ lb_models,
1269
+ search,
1270
+ platform,
1271
+ owner,
1272
+ optimized,
1273
+ ],
1274
+ outputs=ranking,
1275
+ )
1276
+
1277
+ prompt_count.change(
1278
+ on_samples_controls,
1279
+ inputs=[sm_dataset, sm_models, prompt_count, seed_state],
1280
+ outputs=gallery,
1281
+ )
1282
+ shuffle_button.click(
1283
+ on_shuffle,
1284
+ inputs=[sm_dataset, sm_models, prompt_count, seed_state],
1285
+ outputs=[seed_state, gallery],
1286
+ )
1287
+
1288
 
1289
  def render_about():
1290
  with gr.Row(elem_classes="about-layout", equal_height=False):