begumcig commited on
Commit
4527ab9
·
1 Parent(s): 4b9f11d

feat: add AA and Arenaai results, fix some ui features

Browse files
app.py CHANGED
@@ -2023,6 +2023,29 @@ footer, .footer { display: none !important; }
2023
  color: var(--pruna-text-primary) !important;
2024
  font-size: 1rem;
2025
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2026
 
2027
  .options,
2028
  ul.options,
@@ -2271,6 +2294,72 @@ def load_oneig_dataframe(path):
2271
  return df
2272
 
2273
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2274
  def load_qwen_combined_dataframe(path):
2275
  """Load the combined Qwen Image Bench metrics table."""
2276
  df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
@@ -2288,7 +2377,6 @@ def load_qwen_combined_dataframe(path):
2288
  "P-Judge Overall",
2289
  "Rapidata Elo",
2290
  "Datapoint Elo",
2291
- "Benchmark.ai Elo",
2292
  ]:
2293
  if column in df.columns:
2294
  df[column] = pd.to_numeric(df[column], errors="coerce")
@@ -2369,8 +2457,18 @@ qwen_path = _resolve_data_path(
2369
  data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
2370
  space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
2371
  )
 
 
 
 
 
 
 
 
2372
 
2373
  qwen_df = load_qwen_combined_dataframe(qwen_path)
 
 
2374
  qwen_display_columns = [
2375
  col
2376
  for col in [
@@ -2389,12 +2487,19 @@ aa_display_columns = [
2389
  col
2390
  for col in [
2391
  "Model",
2392
- "Benchmark.ai Elo",
2393
- "Median Generation Time (s)",
2394
- "Min Generation Time (s)",
2395
  "Price / Image (USD)",
2396
  ]
2397
- if col in qwen_df.columns
 
 
 
 
 
 
 
 
 
2398
  ]
2399
 
2400
  oneig_samples = load_sample_comparison_data(oneig_combined_dir)
@@ -2439,7 +2544,47 @@ metrics = [
2439
  {
2440
  "id": "aa_elo",
2441
  "name": "Artificial Analysis ELO Metric",
2442
- "column": "Benchmark.ai Elo",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2443
  },
2444
  ]
2445
 
@@ -2467,7 +2612,20 @@ oneig_metric_ids = _metric_ids_for(
2467
  "datapoint_elo_object",
2468
  ],
2469
  )
2470
- aa_metric_ids = _metric_ids_for(qwen_df, ["aa_elo"])
 
 
 
 
 
 
 
 
 
 
 
 
 
2471
 
2472
  datasets = [
2473
  {
@@ -2496,14 +2654,21 @@ datasets = [
2496
  },
2497
  {
2498
  "id": "artificial_analysis",
2499
- "name": "Artificial Analysis Dataset",
2500
- "data": qwen_df,
2501
  "columns": aa_display_columns,
2502
  "metric_ids": aa_metric_ids,
2503
- "note": (
2504
- "> Ranked by Artificial Analysis Elo from the evaluation table."
2505
- ),
2506
- "samples": qwen_samples,
 
 
 
 
 
 
 
2507
  },
2508
  ]
2509
  datasets = [dataset for dataset in datasets if dataset["metric_ids"]]
@@ -2512,11 +2677,7 @@ DEFAULT_DATASET_ID = next(
2512
  (dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
2513
  datasets[0]["id"] if datasets else None,
2514
  )
2515
- DEFAULT_METRIC_ID = (
2516
- "datapoint_elo"
2517
- if DEFAULT_DATASET_ID == "qwen" and "datapoint_elo" in qwen_metric_ids
2518
- else (datasets[0]["metric_ids"][0] if datasets else None)
2519
- )
2520
 
2521
 
2522
  custom_head = """
 
2023
  color: var(--pruna-text-primary) !important;
2024
  font-size: 1rem;
2025
  }
2026
+ .pareto-panel-host {
2027
+ display: flex;
2028
+ flex-direction: column;
2029
+ gap: 1.5rem;
2030
+ }
2031
+ .pareto-metric-block {
2032
+ padding-bottom: 0.5rem;
2033
+ border-bottom: 1px solid var(--pruna-hairline);
2034
+ }
2035
+ .pareto-metric-block:last-child {
2036
+ border-bottom: none;
2037
+ }
2038
+ .pareto-metric-title {
2039
+ margin: 0 0 0.75rem;
2040
+ color: var(--pruna-text-primary);
2041
+ font-size: 1.05rem;
2042
+ font-weight: 600;
2043
+ }
2044
+ .pareto-subhead {
2045
+ margin: 0 0 0.35rem;
2046
+ color: var(--pruna-text-muted);
2047
+ font-size: 0.9rem;
2048
+ }
2049
 
2050
  .options,
2051
  ul.options,
 
2294
  return df
2295
 
2296
 
2297
+ def load_artificial_analysis_dataframe(path):
2298
+ """Load the Artificial Analysis text-to-image leaderboard."""
2299
+ df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
2300
+ df = df.rename(
2301
+ columns={
2302
+ "model": "Model",
2303
+ "elo": "Artificial Analysis Elo",
2304
+ "price_per_image_usd": "Price / Image (USD)",
2305
+ }
2306
+ )
2307
+ df["Model"] = df["Model"].astype(str).str.strip()
2308
+ for column in ["Artificial Analysis Elo", "Price / Image (USD)"]:
2309
+ if column in df.columns:
2310
+ df[column] = pd.to_numeric(df[column], errors="coerce")
2311
+ if "Artificial Analysis Elo" in df.columns:
2312
+ df = df.sort_values(
2313
+ by="Artificial Analysis Elo", ascending=False, na_position="last"
2314
+ )
2315
+ numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
2316
+ for col in numeric_cols:
2317
+ df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
2318
+ return df.reset_index(drop=True)
2319
+
2320
+
2321
+ ARENA_CATEGORY_COLUMNS = {
2322
+ "product_branding_commercial_design_elo": "Arena Branding / Commercial Elo",
2323
+ "3d_imaging_and_modeling_elo": "Arena 3D Imaging Elo",
2324
+ "cartoon_anime_and_fantasy_elo": "Arena Cartoon / Anime Elo",
2325
+ "photorealistic_and_cinematic_imagery_elo": "Arena Photorealistic Elo",
2326
+ "art_elo": "Arena Art Elo",
2327
+ "portraits_elo": "Arena Portraits Elo",
2328
+ "text_rendering_elo": "Arena Text Rendering Elo",
2329
+ }
2330
+
2331
+
2332
+ def load_arena_ai_dataframe(path):
2333
+ """Load the Arena AI text-to-image leaderboard (overall + category Elos)."""
2334
+ df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
2335
+ rename_map = {"model": "Model", "elo": "Arena Elo"}
2336
+ rename_map.update(
2337
+ {
2338
+ source: label
2339
+ for source, label in ARENA_CATEGORY_COLUMNS.items()
2340
+ if source in df.columns
2341
+ }
2342
+ )
2343
+ df = df.rename(columns=rename_map)
2344
+ df["Model"] = df["Model"].astype(str).str.strip()
2345
+ score_columns = [
2346
+ column
2347
+ for column in ["Arena Elo", *ARENA_CATEGORY_COLUMNS.values()]
2348
+ if column in df.columns
2349
+ ]
2350
+ for column in score_columns:
2351
+ df[column] = pd.to_numeric(df[column], errors="coerce")
2352
+ if "Arena Elo" in df.columns:
2353
+ df = df.sort_values(by="Arena Elo", ascending=False, na_position="last")
2354
+ numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
2355
+ for col in numeric_cols:
2356
+ df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
2357
+ ordered = ["Model", *score_columns]
2358
+ return df[[column for column in ordered if column in df.columns]].reset_index(
2359
+ drop=True
2360
+ )
2361
+
2362
+
2363
  def load_qwen_combined_dataframe(path):
2364
  """Load the combined Qwen Image Bench metrics table."""
2365
  df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
 
2377
  "P-Judge Overall",
2378
  "Rapidata Elo",
2379
  "Datapoint Elo",
 
2380
  ]:
2381
  if column in df.columns:
2382
  df[column] = pd.to_numeric(df[column], errors="coerce")
 
2457
  data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
2458
  space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
2459
  )
2460
+ aa_path = _resolve_data_path(
2461
+ data_dir / "artificial_analysis_text_to_image_leaderboard.csv",
2462
+ space_root.parent / "artificial_analysis_text_to_image_leaderboard.csv",
2463
+ )
2464
+ arena_path = _resolve_data_path(
2465
+ data_dir / "arena_ai_text_to_image_leaderboard.csv",
2466
+ space_root.parent / "arena_ai_text_to_image_leaderboard.csv",
2467
+ )
2468
 
2469
  qwen_df = load_qwen_combined_dataframe(qwen_path)
2470
+ aa_df = load_artificial_analysis_dataframe(aa_path)
2471
+ arena_df = load_arena_ai_dataframe(arena_path)
2472
  qwen_display_columns = [
2473
  col
2474
  for col in [
 
2487
  col
2488
  for col in [
2489
  "Model",
2490
+ "Artificial Analysis Elo",
 
 
2491
  "Price / Image (USD)",
2492
  ]
2493
+ if col in aa_df.columns
2494
+ ]
2495
+ arena_display_columns = [
2496
+ col
2497
+ for col in [
2498
+ "Model",
2499
+ "Arena Elo",
2500
+ *ARENA_CATEGORY_COLUMNS.values(),
2501
+ ]
2502
+ if col in arena_df.columns
2503
  ]
2504
 
2505
  oneig_samples = load_sample_comparison_data(oneig_combined_dir)
 
2544
  {
2545
  "id": "aa_elo",
2546
  "name": "Artificial Analysis ELO Metric",
2547
+ "column": "Artificial Analysis Elo",
2548
+ },
2549
+ {
2550
+ "id": "arena_elo",
2551
+ "name": "Arena ELO - Overall Metric",
2552
+ "column": "Arena Elo",
2553
+ },
2554
+ {
2555
+ "id": "arena_branding",
2556
+ "name": "Arena ELO - Branding / Commercial Metric",
2557
+ "column": "Arena Branding / Commercial Elo",
2558
+ },
2559
+ {
2560
+ "id": "arena_3d",
2561
+ "name": "Arena ELO - 3D Imaging Metric",
2562
+ "column": "Arena 3D Imaging Elo",
2563
+ },
2564
+ {
2565
+ "id": "arena_cartoon",
2566
+ "name": "Arena ELO - Cartoon / Anime Metric",
2567
+ "column": "Arena Cartoon / Anime Elo",
2568
+ },
2569
+ {
2570
+ "id": "arena_photo",
2571
+ "name": "Arena ELO - Photorealistic Metric",
2572
+ "column": "Arena Photorealistic Elo",
2573
+ },
2574
+ {
2575
+ "id": "arena_art",
2576
+ "name": "Arena ELO - Art Metric",
2577
+ "column": "Arena Art Elo",
2578
+ },
2579
+ {
2580
+ "id": "arena_portraits",
2581
+ "name": "Arena ELO - Portraits Metric",
2582
+ "column": "Arena Portraits Elo",
2583
+ },
2584
+ {
2585
+ "id": "arena_text",
2586
+ "name": "Arena ELO - Text Rendering Metric",
2587
+ "column": "Arena Text Rendering Elo",
2588
  },
2589
  ]
2590
 
 
2612
  "datapoint_elo_object",
2613
  ],
2614
  )
2615
+ aa_metric_ids = _metric_ids_for(aa_df, ["aa_elo"])
2616
+ arena_metric_ids = _metric_ids_for(
2617
+ arena_df,
2618
+ [
2619
+ "arena_elo",
2620
+ "arena_branding",
2621
+ "arena_3d",
2622
+ "arena_cartoon",
2623
+ "arena_photo",
2624
+ "arena_art",
2625
+ "arena_portraits",
2626
+ "arena_text",
2627
+ ],
2628
+ )
2629
 
2630
  datasets = [
2631
  {
 
2654
  },
2655
  {
2656
  "id": "artificial_analysis",
2657
+ "name": "Artificial Analysis",
2658
+ "data": aa_df,
2659
  "columns": aa_display_columns,
2660
  "metric_ids": aa_metric_ids,
2661
+ "note": "",
2662
+ "samples": None,
2663
+ },
2664
+ {
2665
+ "id": "arena_ai",
2666
+ "name": "Arena AI",
2667
+ "data": arena_df,
2668
+ "columns": arena_display_columns,
2669
+ "metric_ids": arena_metric_ids,
2670
+ "note": "",
2671
+ "samples": None,
2672
  },
2673
  ]
2674
  datasets = [dataset for dataset in datasets if dataset["metric_ids"]]
 
2677
  (dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
2678
  datasets[0]["id"] if datasets else None,
2679
  )
2680
+ DEFAULT_METRIC_ID = None
 
 
 
 
2681
 
2682
 
2683
  custom_head = """
data/arena_ai_text_to_image_leaderboard.csv ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model,elo,product_branding_commercial_design_elo,3d_imaging_and_modeling_elo,cartoon_anime_and_fantasy_elo,photorealistic_and_cinematic_imagery_elo,art_elo,portraits_elo,text_rendering_elo
2
+ gpt-image-2 (medium),1381,1391,1361,1397,1380,1370,1429,1425
3
+ mai-image-2.6-preview,1336,1344,1362,1355,1325,1345,1365,1374
4
+ grok-imagine-image-2.0 (low),1316,1309,1268,1321,1327,1306,1371,1353
5
+ reve-2.1,1302,1314,1274,1310,1302,1289,1317,1341
6
+ muse-image,1282,1269,1255,1289,1293,1275,1313,1304
7
+ reve-2.0,1270,1279,1220,1275,1280,1259,1283,1297
8
+ gemini-3.1-flash-image (nano-banana-2) [web-search],1264,1271,1260,1268,1270,1251,1266,1295
9
+ seedream-5.0-pro,1258,1234,1246,1273,1280,1269,1291,1255
10
+ qwen-image-3.0-pro,1257,1258,1240,1271,1257,1257,1273,1289
11
+ mai-image-2.5,1256,1261,1248,1270,1252,1278,1259,1284
12
+ gemini-3.1-flash-lite-image (nano-banana-2-lite),1251,1250,1225,1258,1260,1257,1248,1275
13
+ gemini-3-pro-image-2k (nano-banana-pro),1246,1246,1231,1242,1260,1225,1258,1270
14
+ gpt-image-1.5-high-fidelity,1239,1241,1216,1243,1249,1224,1261,1254
15
+ gemini-3-pro-image-preview (nano-banana-pro),1232,1237,1209,1238,1243,1216,1236,1254
16
+ ideogram-4.0-quality,1204,1224,1180,1195,1203,1199,1201,1238
17
+ qwen-image-2.0-pro-2026-06-22,1191,1187,1198,1191,1205,1177,1216,1201
18
+ uni-1.1-max,1188,1197,1172,1194,1188,1195,1158,1217
19
+ mai-image-2,1183,1175,1179,1180,1203,1177,1207,1175
20
+ uni-1.1,1181,1181,1167,1189,1180,1188,1176,1202
21
+ Cosmos3-Super-Text2Image (Agentic),1175,1162,1170,1191,1189,1195,1212,1151
22
+ grok-imagine-image,1171,1175,1176,1177,1179,1179,1187,1185
23
+ recraft-v4.1-utility-pro,1169,1167,1147,1177,1180,1178,1169,1162
24
+ flux-2-max,1162,1165,1161,1169,1161,1170,1157,1167
25
+ grok-imagine-image-pro,1161,1168,1165,1163,1169,1165,1172,1182
26
+ flux-2-flex,1157,1173,1172,1162,1147,1156,1145,1172
27
+ flux-2-pro,1155,1157,1150,1161,1152,1163,1146,1157
28
+ Cosmos3-Super-Text2Image,1155,1129,1166,1177,1155,1173,1176,1122
29
+ reve-v1.5,1154,1182,1143,1141,1147,1144,1149,1185
30
+ hunyuan-image-3.0,1151,1153,1168,1163,1152,1153,1146,1150
31
+ gemini-2.5-flash-image-preview (nano-banana),1150,1152,1142,1145,1157,1145,1168,1154
32
+ imagen-ultra-4.0-generate-001,1148,1151,1141,1141,1156,1128,1168,1148
33
+ seedream-4.5,1147,1140,1147,1149,1166,1154,1166,1141
34
+ flux-2-dev,1145,1155,1149,1144,1143,1150,1133,1159
35
+ seedream-4-2k,1140,1134,1150,1149,1151,1164,1142,1143
36
+ seedream-5.0-lite,1137,1129,1138,1140,1154,1137,1150,1134
37
+ wan2.6-t2i,1136,1148,1146,1146,1128,1156,1123,1149
38
+ recraft-v4.1-pro,1130,1138,1111,1122,1143,1154,1144,1156
39
+ imagen-4.0-generate-001,1129,1129,1123,1120,1136,1110,1145,1123
40
+ qwen-image-2512,1125,1128,1135,1125,1135,1133,1143,1122
41
+ krea-2-medium,1122,1119,1132,1140,1127,1152,1148,1124
42
+ wan2.5-t2i-preview,1117,1138,1135,1120,1102,1120,1094,1142
43
+ hidream-o1-image,1117,1128,1121,1121,1111,1122,1106,1135
44
+ seedream-4-fal,1116,1128,1138,1109,1121,1119,1116,1124
45
+ gpt-image-1,1115,1115,1114,1115,1115,1115,1115,1115
46
+ recraft-v4,1114,1126,1076,1111,1119,1123,1119,1139
47
+ seedream-4-high-res-fal,1113,1113,1121,1109,1129,1112,1133,1110
48
+ krea-2-turbo,1111,1109,1133,1122,1123,1139,1138,1097
49
+ gpt-image-1-mini,1109,1109,1108,1108,1114,1113,1112,1104
50
+ krea-2-large,1107,1104,1113,1112,1121,1116,1117,1111
51
+ wan2.7-image-pro,1103,1114,1080,1080,1123,1079,1127,1118
52
+ wan2.7-image,1100,1110,1082,1075,1117,1075,1125,1114
53
+ mai-image-1,1093,1086,1096,1100,1103,1104,1095,1070
54
+ seedream-3,1082,1070,1094,1081,1102,1092,1102,1052
55
+ z-image-turbo,1082,1060,1066,1067,1115,1074,1140,1058
56
+ flux-1-kontext-max,1074,1082,1079,1079,1083,1081,1085,1071
57
+ flux-2-klein-9b,1070,1070,1078,1064,1090,1079,1092,1057
58
+ qwen-image-prompt-extend,1061,1087,1082,1062,1047,1055,1051,1082
59
+ flux-1-kontext-pro,1059,1068,1062,1058,1066,1067,1065,1053
60
+ imagen-3.0-generate-002,1058,1057,1046,1048,1075,1059,1077,1045
61
+ qwen-image,1057,1068,1064,1052,1062,1055,1076,1047
62
+ ideogram-v3-quality,1049,1082,1029,1027,1051,1039,1052,1070
63
+ photon,1035,1052,1033,1026,1035,1046,1023,1041
64
+ p-image,1034,1031,1031,1014,1074,1041,1100,1021
65
+ flux-2-klein-4b,1030,1026,1030,1023,1047,1028,1054,1015
66
+ runway-gen4,1025,1027,1013,1008,1041,1019,1022,1019
67
+ recraft-v3,1021,1027,1011,978,1054,991,1040,1005
68
+ flux-1.1-pro,1016,1012,1051,1016,1044,1044,1043,995
69
+ lucid-origin,1013,1020,1015,1007,1035,1031,1043,1005
70
+ ideogram-v2,1013,1054,990,995,1019,1017,1019,1046
71
+ glm-image,1010,1036,1018,1003,1000,1013,1000,1033
72
+ gemini-2.0-flash-preview-image-generation,975,1019,976,954,963,965,961,1007
73
+ flux-1-dev-fp8,969,991,978,957,990,978,999,963
74
+ dall-e-3,968,987,987,988,940,1010,924,969
75
+ flux-1-kontext-dev,940,966,943,924,935,927,944,953
76
+ stable-diffusion-v35-large,938,937,964,946,928,998,941,921
77
+ bagel,898,887,866,885,923,911,926,875
data/artificial_analysis_text_to_image_leaderboard.csv ADDED
@@ -0,0 +1,146 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model,elo,price_per_image_usd
2
+ GPT Image 2 (high),1370,0.211
3
+ Reve 2.1,1324,0.2
4
+ Nano Banana 2 (Gemini 3.1 Flash Image Preview),1322,0.067
5
+ GPT Image 1.5 (high),1312,0.133
6
+ MAI-Image-2.5,1306,0.0481
7
+ Nano Banana Pro (Gemini 3 Pro Image),1298,0.134
8
+ Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image),1292,0.0336
9
+ Seedream 5.0 Pro,1281,0.09
10
+ grok-imagine-image-quality,1236,0.05
11
+ Qwen Image 2.0 Pro (2026-04-22),1235,0.075
12
+ FLUX.2 [max],1231,0.07
13
+ MAI-Image-2.5-Flash,1230,0.02
14
+ HiDream-O1-Image-1.5,1227,0.08
15
+ Seedream 4.0,1224,0.03
16
+ Luma UNI 1 Max,1223,0.1
17
+ FLUX.2 [flex],1222,0.06
18
+ Krea 2 Medium Turbo,1222,0.015
19
+ Krea 2 Large,1220,0.06
20
+ grok-imagine-image,1218,0.02
21
+ Recraft V4.1 Utility,1217,0.035
22
+ Ideogram 4.0,1217,0.06
23
+ Recraft V4.1 Utility Pro,1217,0.21
24
+ Ideogram 4.0 (Quality),1214,0.1
25
+ Wan2.6 Text to Image,1213,0.03
26
+ Krea 2 Medium,1212,0.03
27
+ MAI-Image-2,1210,0.035
28
+ FLUX.2 [pro],1208,0.03
29
+ Wan 2.6 Image,1206,0.03
30
+ Seedream 4.5,1205,0.04
31
+ GPT Image 1 (high),1205,0.167
32
+ Luma UNI 1,1205,0.0404
33
+ FLUX.2 [dev] Turbo,1199,0.008
34
+ FLUX.2 [dev],1199,0.012
35
+ Seedream 5.0 Lite,1199,0.035
36
+ Ideogram 4.0 Fast (Quality),1199,0.0175
37
+ P-Image-Ideogram (High),1198,0.015
38
+ Recraft V4 Pro,1195,0.25
39
+ Recraft V4.1,1191,0.035
40
+ Nano Banana (Gemini 2.5 Flash Image),1189,0.039
41
+ Recraft V4.1 Pro,1189,0.21
42
+ Imagen 4 Ultra,1189,0.06
43
+ Ideogram 4.0 Instant,1187,0.0075
44
+ Wan 2.7 Pro,1184,0.064
45
+ Recraft V4,1182,0.04
46
+ MAI-Image-2-Efficient,1182,0.022
47
+ P-Image-Ideogram (Medium),1181,0.01
48
+ FLUX.2 [dev] Flash,1179,0.005
49
+ HiDream-O1-Image,1176,
50
+ Ideogram 4.0 Fast,1176,0.0105
51
+ ImagineArt 2.0,1173,0.03
52
+ Qwen Image Max 2512,1173,0.02
53
+ Wan 2.7,1170,0.026
54
+ ImagineArt 1.5 Preview,1165,0.03
55
+ P-Image-Ideogram (Low),1165,0.0075
56
+ Seedream 3.0,1160,0.03
57
+ Vivago 2.1,1152,0.035
58
+ HunyuanImage 3.0 Instruct (Fal),1151,0.09
59
+ Wan 2.5 Preview,1150,0.021
60
+ FLUX.2 [klein] 9B,1146,0.015
61
+ FLUX.1 Kontext [max],1142,0.08
62
+ Kolors 2.1,1142,0.014
63
+ image-1,1141,0.04
64
+ HunyuanImage 3.0 (Fal),1140,0.1
65
+ Qwen Image 2.0 (2026-03-03),1135,0.035
66
+ Z-Image Turbo,1130,0.005
67
+ Vivago 2.0,1125,
68
+ Imagen 3 (v002),1124,0.04
69
+ Lucid Origin Ultra,1122,0.0867
70
+ Eigen Image,1122,0.025
71
+ Imagen 4 Standard,1120,0.04
72
+ Reve Image (Halfmoon),1116,
73
+ Kling Image 3.0 Omni,1115,0.028
74
+ Vidu Q2,1112,0.03
75
+ P-Image-Ideogram (Very Low),1111,0.003
76
+ Qwen Image Plus 2601,1110,0.03
77
+ GPT Image 1 Mini (medium),1110,0.011
78
+ Lucid Origin Fast,1110,0.0179
79
+ FLUX.1 Kontext [pro],1109,0.04
80
+ Dreamina 3.1,1108,0.03
81
+ FLUX1.1 [pro] Ultra,1104,0.06
82
+ Ideogram 3.0,1101,0.06
83
+ FLUX.2 [klein] Base 9B,1099,0.011
84
+ Imagen 4 Fast,1098,0.02
85
+ P-Image,1093,0.005
86
+ FLUX1.1 [pro],1092,0.04
87
+ Midjourney v7 Alpha,1092,
88
+ Ideogram v2,1084,0.08
89
+ FLUX.1 [pro],1083,0.05
90
+ Qwen Image,1082,0.02
91
+ SRPO,1080,0.026
92
+ HunyuanImage 2.1,1078,0.1
93
+ Midjourney v6,1076,
94
+ Ideogram v2 Turbo,1075,0.05
95
+ FIBO,1075,0.04
96
+ Recraft V3,1075,0.04
97
+ Luma Photon,1074,0.019
98
+ HiDream-O1-Image-Dev,1071,0.005
99
+ Image-01,1070,0.01
100
+ HiDream-I1-Dev,1070,0.024
101
+ GLM-Image,1068,0.05
102
+ Z-Image Base,1064,0.01
103
+ MAI Image 1,1064,
104
+ HiDream-I1-Fast,1063,0.012
105
+ FLUX.2 [klein] 4B,1057,0.014
106
+ Midjourney v6.1,1057,
107
+ Infinity 8B,1056,0.0017
108
+ LongCat Image,1053,0.13
109
+ Stable Diffusion 3 Large,1042,
110
+ FLUX.1 [dev],1041,0.025
111
+ Phoenix 1.0 Ultra,1041,0.0762
112
+ FLUX.1 Krea [dev],1036,0.025
113
+ Stable Diffusion 3.5 Large Turbo,1034,0.04
114
+ Stable Diffusion 3.5 Large,1034,0.065
115
+ Phoenix 0.9 Ultra,1028,0.0762
116
+ Ideogram v2a Turbo,1028,0.025
117
+ Ideogram v2a,1027,0.04
118
+ Playground v3 (beta),1015,0.0105
119
+ Krea 1,1013,
120
+ Phoenix 1.0 Fast,1007,0.0194
121
+ Luma Photon Flash,1004,0.005
122
+ FLUX.1 [schnell],1000,0.003
123
+ Step Image Edit 2,1000,0.003
124
+ Runway Gen-4 Image,992,0.08
125
+ Recraft 20B,984,0.022
126
+ Lumina Image v2,978,0.075
127
+ Playground v2.5,972,0.08
128
+ FLUX.2 [klein] Base 4B,972,0.009
129
+ DALLE 3,968,0.04
130
+ DALLE 3 HD,967,0.08
131
+ Stable Diffusion 3.5 Medium,963,0.02
132
+ Sana Sprint 1.6B,939,0.0015
133
+ Amazon Titan G1 v2 (Standard),923,0.01
134
+ Stable Diffusion 3 Medium,922,0.035
135
+ Stable Diffusion 3 Large Turbo,917,
136
+ Amazon Titan G1 (Standard),914,0.01
137
+ Stable Diffusion 1.6,914,0.009
138
+ OmniGen V2,911,0.15
139
+ SDXL Lightning,909,0.0019
140
+ Bagel,908,0.1
141
+ Bria 3.2,906,0.04
142
+ Stable Diffusion XL 1.0,884,0.009
143
+ Stable Diffusion 2.1,754,0.0039
144
+ DALLE 2,745,0.02
145
+ Janus Pro,720,0.0
146
+ Stable Diffusion 1.5,665,0.044
model_display.py ADDED
@@ -0,0 +1,251 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """UI-only model display names.
2
+
3
+ Raw CSV / sample IDs stay unchanged; this maps aliases to a shared label
4
+ shown in leaderboards, Pareto plots, dropdowns, and sample headers.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import re
10
+
11
+ # Explicit aliases (any leaderboard / sample id) -> shared display label.
12
+ MODEL_DISPLAY_NAMES = {
13
+ # FLUX
14
+ "flux_2_pro": "FLUX.2 [pro]",
15
+ "FLUX.2 [pro]": "FLUX.2 [pro]",
16
+ "flux_2_max": "FLUX.2 [max]",
17
+ "FLUX.2 [max]": "FLUX.2 [max]",
18
+ "flux_2_flex": "FLUX.2 [flex]",
19
+ "FLUX.2 [flex]": "FLUX.2 [flex]",
20
+ "flux_2_dev": "FLUX.2 [dev]",
21
+ "FLUX.2 [dev]": "FLUX.2 [dev]",
22
+ "FLUX.2 [dev] Turbo": "FLUX.2 [dev] Turbo",
23
+ "FLUX.2 [dev] Flash": "FLUX.2 [dev] Flash",
24
+ "flux_1_1_pro": "FLUX1.1 [pro]",
25
+ "FLUX1.1 [pro]": "FLUX1.1 [pro]",
26
+ "flux_1_1_pro_ultra": "FLUX1.1 [pro] Ultra",
27
+ "FLUX1.1 [pro] Ultra": "FLUX1.1 [pro] Ultra",
28
+ "flux_dev": "FLUX.1 [dev]",
29
+ "FLUX.1 [dev]": "FLUX.1 [dev]",
30
+ "flux_schnell": "FLUX.1 [schnell]",
31
+ "FLUX.1 [schnell]": "FLUX.1 [schnell]",
32
+ "flux_krea": "FLUX.1 Krea [dev]",
33
+ "FLUX.1 Krea [dev]": "FLUX.1 Krea [dev]",
34
+ "FLUX.1 [pro]": "FLUX.1 [pro]",
35
+ "FLUX.1 Kontext [pro]": "FLUX.1 Kontext [pro]",
36
+ "FLUX.1 Kontext [max]": "FLUX.1 Kontext [max]",
37
+ # GPT Image
38
+ "gpt_image_2": "GPT Image 2",
39
+ "GPT Image 2 (high)": "GPT Image 2",
40
+ "gpt_image_1_5": "GPT Image 1.5",
41
+ "GPT Image 1.5 (high)": "GPT Image 1.5",
42
+ "gpt_image_1": "GPT Image 1",
43
+ "GPT Image 1 (high)": "GPT Image 1",
44
+ "GPT Image 1 Mini (medium)": "GPT Image 1 Mini",
45
+ # Nano Banana / Gemini
46
+ "nano_banana_2_0": "Nano Banana 2",
47
+ "nano_banana_2": "Nano Banana 2",
48
+ "Nano Banana 2 (Gemini 3.1 Flash Image Preview)": "Nano Banana 2",
49
+ "Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)": "Nano Banana 2 Lite",
50
+ "nano_banana_pro": "Nano Banana Pro",
51
+ "Nano Banana Pro (Gemini 3 Pro Image)": "Nano Banana Pro",
52
+ "Nano Banana (Gemini 2.5 Flash Image)": "Nano Banana",
53
+ # Seedream
54
+ "seedream_5_0": "Seedream 5.0",
55
+ "Seedream 5.0 Pro": "Seedream 5.0 Pro",
56
+ "Seedream 5.0 Lite": "Seedream 5.0 Lite",
57
+ "seedream_4_5": "Seedream 4.5",
58
+ "Seedream 4.5": "Seedream 4.5",
59
+ "seedream_4_0": "Seedream 4.0",
60
+ "Seedream 4.0": "Seedream 4.0",
61
+ "seedream_3": "Seedream 3.0",
62
+ "Seedream 3.0": "Seedream 3.0",
63
+ # Qwen
64
+ "qwen_image": "Qwen Image",
65
+ "Qwen Image": "Qwen Image",
66
+ "qwen_image_2_0_pro": "Qwen Image 2.0 Pro",
67
+ "Qwen Image 2.0 Pro (2026-04-22)": "Qwen Image 2.0 Pro",
68
+ "Qwen Image 2.0 (2026-03-03)": "Qwen Image 2.0",
69
+ "qwen_image_2512": "Qwen Image 2512",
70
+ "Qwen Image Max 2512": "Qwen Image 2512",
71
+ "qwen_image_fast": "Qwen Image Fast",
72
+ "Qwen Image Plus 2601": "Qwen Image Plus 2601",
73
+ # Ideogram
74
+ "ideogram_4_0_quality": "Ideogram 4.0 Quality",
75
+ "Ideogram 4.0 (Quality)": "Ideogram 4.0 Quality",
76
+ "Ideogram 4.0": "Ideogram 4.0",
77
+ "Ideogram 4.0 Fast": "Ideogram 4.0 Fast",
78
+ "Ideogram 4.0 Fast (Quality)": "Ideogram 4.0 Fast Quality",
79
+ "Ideogram 4.0 Instant": "Ideogram 4.0 Instant",
80
+ "Ideogram 3.0": "Ideogram 3.0",
81
+ # Imagen
82
+ "imagen_4_0": "Imagen 4",
83
+ "imagen_4": "Imagen 4",
84
+ "Imagen 4 Standard": "Imagen 4",
85
+ "imagen_4_0_ultra": "Imagen 4 Ultra",
86
+ "imagen_4_ultra": "Imagen 4 Ultra",
87
+ "Imagen 4 Ultra": "Imagen 4 Ultra",
88
+ "imagen_4_fast": "Imagen 4 Fast",
89
+ "Imagen 4 Fast": "Imagen 4 Fast",
90
+ "Imagen 3 (v002)": "Imagen 3",
91
+ # HiDream
92
+ "hidream_i1_dev": "HiDream-I1 Dev",
93
+ "HiDream-I1-Dev": "HiDream-I1 Dev",
94
+ "hidream_i1_fast": "HiDream-I1 Fast",
95
+ "HiDream-I1-Fast": "HiDream-I1 Fast",
96
+ "hidream_i1_full": "HiDream-I1 Full",
97
+ "HiDream-O1-Image": "HiDream-O1",
98
+ "HiDream-O1-Image-1.5": "HiDream-O1 1.5",
99
+ "HiDream-O1-Image-Dev": "HiDream-O1 Dev",
100
+ # Reve
101
+ "reve_2_1": "Reve 2.1",
102
+ "Reve 2.1": "Reve 2.1",
103
+ "Reve Image (Halfmoon)": "Reve Image",
104
+ # P-Image
105
+ "p_image": "P-Image",
106
+ "P-Image": "P-Image",
107
+ "p_image_2_ideogram_very_low_1k": "P-Image-Ideogram Very Low 1K",
108
+ "p_image_2_ideogram_very_low_2k": "P-Image-Ideogram Very Low 2K",
109
+ "P-Image-Ideogram (Very Low)": "P-Image-Ideogram Very Low",
110
+ "p_image_2_ideogram_low_1k": "P-Image-Ideogram Low 1K",
111
+ "p_image_2_ideogram_low_2k": "P-Image-Ideogram Low 2K",
112
+ "P-Image-Ideogram (Low)": "P-Image-Ideogram Low",
113
+ "p_image_2_ideogram_medium_1k": "P-Image-Ideogram Medium 1K",
114
+ "p_image_2_ideogram_medium_2k": "P-Image-Ideogram Medium 2K",
115
+ "P-Image-Ideogram (Medium)": "P-Image-Ideogram Medium",
116
+ "p_image_2_ideogram_high_1k": "P-Image-Ideogram High 1K",
117
+ "p_image_2_ideogram_high_2k": "P-Image-Ideogram High 2K",
118
+ "P-Image-Ideogram (High)": "P-Image-Ideogram High",
119
+ # Others overlapping P-Bench
120
+ "z_image": "Z-Image",
121
+ "Z-Image Turbo": "Z-Image Turbo",
122
+ "Z-Image Base": "Z-Image Base",
123
+ "glm_image": "GLM-Image",
124
+ "GLM-Image": "GLM-Image",
125
+ "hunyuanimage_3_0": "HunyuanImage 3.0",
126
+ "hunyuan_image_3": "HunyuanImage 3.0",
127
+ "HunyuanImage 3.0 (Fal)": "HunyuanImage 3.0",
128
+ "HunyuanImage 3.0 Instruct (Fal)": "HunyuanImage 3.0 Instruct",
129
+ "HunyuanImage 2.1": "HunyuanImage 2.1",
130
+ "wan_2_2_image": "Wan 2.2 Image",
131
+ "Wan 2.6 Image": "Wan 2.6 Image",
132
+ "Wan2.6 Text to Image": "Wan 2.6 Text to Image",
133
+ "kling_v2_1": "Kling v2.1",
134
+ "Kling Image 3.0 Omni": "Kling Image 3.0 Omni",
135
+ "juggernaut_base_flux": "Juggernaut Base FLUX",
136
+ "juggernaut_pro_flux": "Juggernaut Pro FLUX",
137
+ "juggernaut_lightning_flux": "Juggernaut Lightning FLUX",
138
+ "bria_4_fibo": "Bria FIBO",
139
+ "Bria 3.2": "Bria 3.2",
140
+ "sdxl": "SDXL 1.0",
141
+ "Stable Diffusion XL 1.0": "SDXL 1.0",
142
+ "sdxl_lightning": "SDXL Lightning",
143
+ "SDXL Lightning": "SDXL Lightning",
144
+ # Arena AI (kebab / arena.ai ids)
145
+ "gpt-image-2 (medium)": "GPT Image 2",
146
+ "gpt-image-1.5-high-fidelity": "GPT Image 1.5",
147
+ "gpt-image-1": "GPT Image 1",
148
+ "gpt-image-1-mini": "GPT Image 1 Mini",
149
+ "mai-image-2.6-preview": "MAI-Image-2.6 Preview",
150
+ "mai-image-2.5": "MAI-Image-2.5",
151
+ "mai-image-2": "MAI-Image-2",
152
+ "mai-image-1": "MAI Image 1",
153
+ "grok-imagine-image-2.0 (low)": "Grok Imagine Image 2.0",
154
+ "grok-imagine-image": "Grok Imagine Image",
155
+ "grok-imagine-image-pro": "Grok Imagine Image Pro",
156
+ "reve-2.1": "Reve 2.1",
157
+ "reve-2.0": "Reve 2.0",
158
+ "reve-v1.5": "Reve 1.5",
159
+ "muse-image": "Muse Image",
160
+ "gemini-3.1-flash-image (nano-banana-2) [web-search]": "Nano Banana 2",
161
+ "gemini-3.1-flash-lite-image (nano-banana-2-lite)": "Nano Banana 2 Lite",
162
+ "gemini-3-pro-image-2k (nano-banana-pro)": "Nano Banana Pro 2K",
163
+ "gemini-3-pro-image-preview (nano-banana-pro)": "Nano Banana Pro",
164
+ "gemini-2.5-flash-image-preview (nano-banana)": "Nano Banana",
165
+ "gemini-2.0-flash-preview-image-generation": "Gemini 2.0 Flash Image",
166
+ "seedream-5.0-pro": "Seedream 5.0 Pro",
167
+ "seedream-5.0-lite": "Seedream 5.0 Lite",
168
+ "seedream-4.5": "Seedream 4.5",
169
+ "seedream-4-2k": "Seedream 4.0 2K",
170
+ "seedream-4-fal": "Seedream 4.0",
171
+ "seedream-4-high-res-fal": "Seedream 4.0 High Res",
172
+ "seedream-3": "Seedream 3.0",
173
+ "qwen-image-3.0-pro": "Qwen Image 3.0 Pro",
174
+ "qwen-image-2.0-pro-2026-06-22": "Qwen Image 2.0 Pro",
175
+ "qwen-image-2512": "Qwen Image 2512",
176
+ "qwen-image-prompt-extend": "Qwen Image Prompt Extend",
177
+ "qwen-image": "Qwen Image",
178
+ "uni-1.1-max": "Luma UNI 1 Max",
179
+ "uni-1.1": "Luma UNI 1",
180
+ "Cosmos3-Super-Text2Image (Agentic)": "Cosmos3 Super Text2Image (Agentic)",
181
+ "Cosmos3-Super-Text2Image": "Cosmos3 Super Text2Image",
182
+ "recraft-v4.1-utility-pro": "Recraft V4.1 Utility Pro",
183
+ "recraft-v4.1-pro": "Recraft V4.1 Pro",
184
+ "recraft-v4": "Recraft V4",
185
+ "recraft-v3": "Recraft V3",
186
+ "flux-2-max": "FLUX.2 [max]",
187
+ "flux-2-flex": "FLUX.2 [flex]",
188
+ "flux-2-pro": "FLUX.2 [pro]",
189
+ "flux-2-dev": "FLUX.2 [dev]",
190
+ "flux-2-klein-9b": "FLUX.2 [klein] 9B",
191
+ "flux-2-klein-4b": "FLUX.2 [klein] 4B",
192
+ "flux-1-kontext-max": "FLUX.1 Kontext [max]",
193
+ "flux-1-kontext-pro": "FLUX.1 Kontext [pro]",
194
+ "flux-1-kontext-dev": "FLUX.1 Kontext [dev]",
195
+ "flux-1.1-pro": "FLUX1.1 [pro]",
196
+ "flux-1-dev-fp8": "FLUX.1 [dev]",
197
+ "hunyuan-image-3.0": "HunyuanImage 3.0",
198
+ "imagen-ultra-4.0-generate-001": "Imagen 4 Ultra",
199
+ "imagen-4.0-generate-001": "Imagen 4",
200
+ "imagen-3.0-generate-002": "Imagen 3",
201
+ "wan2.6-t2i": "Wan 2.6 Text to Image",
202
+ "wan2.5-t2i-preview": "Wan 2.5 Preview",
203
+ "wan2.7-image-pro": "Wan 2.7 Pro",
204
+ "wan2.7-image": "Wan 2.7",
205
+ "ideogram-4.0-quality": "Ideogram 4.0 Quality",
206
+ "ideogram-v3-quality": "Ideogram 3.0",
207
+ "ideogram-v2": "Ideogram v2",
208
+ "krea-2-medium": "Krea 2 Medium",
209
+ "krea-2-turbo": "Krea 2 Medium Turbo",
210
+ "krea-2-large": "Krea 2 Large",
211
+ "hidream-o1-image": "HiDream-O1",
212
+ "z-image-turbo": "Z-Image Turbo",
213
+ "photon": "Luma Photon",
214
+ "p-image": "P-Image",
215
+ "runway-gen4": "Runway Gen-4 Image",
216
+ "lucid-origin": "Lucid Origin",
217
+ "glm-image": "GLM-Image",
218
+ "dall-e-3": "DALLE 3",
219
+ "stable-diffusion-v35-large": "Stable Diffusion 3.5 Large",
220
+ "bagel": "Bagel",
221
+ }
222
+
223
+
224
+ def _prettify_snake_case(model_id: str) -> str:
225
+ parts = [part for part in str(model_id).split("_") if part]
226
+ pretty = []
227
+ for part in parts:
228
+ if part.isdigit():
229
+ pretty.append(part)
230
+ elif re.fullmatch(r"\d+k", part, flags=re.IGNORECASE):
231
+ pretty.append(part.upper())
232
+ elif part.lower() in {"flux", "gpt", "sdxl", "glm"}:
233
+ pretty.append(part.upper())
234
+ else:
235
+ pretty.append(part.capitalize())
236
+ return " ".join(pretty)
237
+
238
+
239
+ def display_model_name(model_id) -> str:
240
+ """Return the shared UI label for a raw model id / source name."""
241
+ if model_id is None:
242
+ return ""
243
+ raw = str(model_id).strip()
244
+ if not raw:
245
+ return ""
246
+ if raw in MODEL_DISPLAY_NAMES:
247
+ return MODEL_DISPLAY_NAMES[raw]
248
+ # Already a human label (spaces / punctuation) — keep as-is.
249
+ if re.search(r"[\s.\[\]()]", raw):
250
+ return raw
251
+ return _prettify_snake_case(raw)
ui.py CHANGED
@@ -7,6 +7,8 @@ import gradio as gr
7
  import pandas as pd
8
  import plotly.graph_objects as go
9
 
 
 
10
  _MASCOT_PATH = Path(__file__).resolve().parent / "pruna-mascot.png"
11
  _MASCOT_DATA_URI = (
12
  "data:image/png;base64,"
@@ -19,6 +21,7 @@ _MASCOT_DATA_URI = (
19
  MAX_COMPARE_MODELS = 4
20
  DEFAULT_COMPARE_PROMPTS = 3
21
  MAX_COMPARE_PROMPTS = 8
 
22
 
23
  ABOUT_OVERVIEW_CONTENT = """
24
  # About P-Bench
@@ -43,8 +46,7 @@ across P-Bench.
43
  2. It generates one image per prompt when the run succeeds. Not every model
44
  has every prompt or every metric.
45
  3. Quality is scored automatically (OneIG alignment, P-Judger) and, where
46
- available, by human preference (Datapoint Elo, Rapidata Elo) or by
47
- Artificial Analysis Elo.
48
  4. Price per image and generation time are joined from the evaluation table.
49
 
50
  ## Current datasets
@@ -59,10 +61,6 @@ Prompt-image **alignment** on anime / stylization, human / portrait, and
59
  general object prompts (100 prompts each). This is the alignment slice of
60
  OneIG, not the full suite. Alignment Overall is the mean of the category
61
  scores that exist for that row.
62
-
63
- ### Artificial Analysis Dataset
64
- Artificial Analysis Elo from the evaluation table. Rapidata is a **metric**,
65
- not a dataset.
66
  """
67
 
68
  ABOUT_DETAILS_CONTENT = """
@@ -79,8 +77,6 @@ ABOUT_DETAILS_CONTENT = """
79
  - **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.
80
  Rapidata rejects prompts over 400 characters, so this Elo is on a subset
81
  of each suite (see Setup). Rapidata is not a dataset.
82
- - **Artificial Analysis Elo**: preference Elo from Artificial Analysis, shown
83
- as its own dataset.
84
  - **Generation time**: median and minimum generation time in seconds, as
85
  reported in the evaluation table. This is not a p95, and we do not state
86
  warm vs cold or concurrent load.
@@ -186,8 +182,23 @@ def _item(items, item_id):
186
  return items[0] if items else None
187
 
188
 
189
- def _dataset_choices(datasets):
190
- return [(dataset["name"], dataset["id"]) for dataset in datasets]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
191
 
192
 
193
  def _metric_choices(datasets, metrics, dataset_id):
@@ -205,11 +216,21 @@ def _metric_choices(datasets, metrics, dataset_id):
205
 
206
 
207
  def _coerce_metric(datasets, metrics, dataset_id, metric_id):
 
 
 
 
 
208
  choices = _metric_choices(datasets, metrics, dataset_id)
209
  ids = [choice[1] for choice in choices]
210
  if metric_id in ids:
211
  return metric_id
212
- return ids[0] if ids else None
 
 
 
 
 
213
 
214
 
215
  def _model_choices(datasets, dataset_id):
@@ -217,42 +238,92 @@ def _model_choices(datasets, dataset_id):
217
  data = dataset.get("data") if dataset else None
218
  if data is None or "Model" not in getattr(data, "columns", []):
219
  return []
220
- return sorted(data["Model"].dropna().astype(str).unique().tolist())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
221
 
222
 
223
  def _view_title(datasets, metrics, dataset_id, metric_id):
224
  dataset = _item(datasets, dataset_id)
225
- metric = _item(metrics, metric_id)
226
  dataset_name = dataset["name"] if dataset else "Dataset"
227
- metric_name = metric["name"] if metric else "Metric"
228
- return f"{dataset_name} | {metric_name}"
 
 
 
 
 
 
 
 
 
 
 
229
 
230
 
231
  def _columns_for_metric(dataset, metric_column):
232
- columns = list(dataset.get("columns") or [])
233
- if metric_column and metric_column not in columns:
234
- identity = {"Model", "Platform", "Endpoint Owner", "Optimized"}
235
- insert_at = 0
236
- for index, column in enumerate(columns):
237
- if column in identity:
238
- insert_at = index + 1
239
- columns.insert(insert_at, metric_column)
240
- return columns
 
 
 
 
241
 
242
 
243
  def resolve_view(datasets, metrics, dataset_id, metric_id):
244
  dataset = _item(datasets, dataset_id)
245
- metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
246
- metric = _item(metrics, metric_id)
247
- if not dataset or not metric:
248
  return None
 
 
 
 
 
 
 
 
 
 
 
 
 
249
  return {
250
  "dataset": dataset,
251
  "metric": metric,
252
- "title": _view_title(datasets, metrics, dataset["id"], metric["id"]),
 
253
  "data": dataset["data"],
254
- "columns": _columns_for_metric(dataset, metric["column"]),
255
- "score_column": metric["column"],
 
256
  "samples": dataset.get("samples"),
257
  "note": dataset.get("note"),
258
  }
@@ -272,6 +343,8 @@ def _format_leaderboard_cell(column, value):
272
  except (TypeError, ValueError):
273
  return escape(str(value))
274
  if label in {"model", "platform", "endpoint owner", "optimized"}:
 
 
275
  return escape(str(value))
276
  try:
277
  number = float(value)
@@ -384,6 +457,11 @@ def _filter_leaderboard(data, search_term, platform, owner, optimized, models=No
384
  matches |= filtered[column].astype(str).str.contains(
385
  search_term, case=False, na=False
386
  )
 
 
 
 
 
387
  filtered = filtered[matches]
388
 
389
  for column, values in [
@@ -467,7 +545,14 @@ def _display_label(column):
467
  "P-Judge Overall": "P-Judger (Pruna)",
468
  "Datapoint Elo": "Datapoint Elo",
469
  "Rapidata Elo": "Rapidata Elo",
470
- "Benchmark.ai Elo": "Artificial Analysis Elo",
 
 
 
 
 
 
 
471
  "Raw Win Rate": "Raw win rate",
472
  "Median Generation Time (s)": "Median generation time",
473
  "Min Generation Time (s)": "Min generation time",
@@ -544,8 +629,12 @@ def _build_pareto_figure(
544
  scores = scatter[score_column].astype(float).tolist()
545
  on_frontier = _pareto_frontier_mask(x_values, scores)
546
 
547
- dominated = scatter.loc[[not flag for flag in on_frontier]]
548
- frontier = scatter.loc[on_frontier].sort_values(x_column)
 
 
 
 
549
  hover = (
550
  "<b>%{text}</b><br>"
551
  f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"
@@ -661,13 +750,37 @@ def _pareto_pair(data, score_column):
661
  )
662
 
663
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
664
  def _samples_html(samples, selected_models, num_prompts, seed=0):
665
  if not samples:
666
- return (
667
- '<div class="compare-empty">'
668
- "Sample comparison is not available for this dataset yet."
669
- "</div>"
670
- )
671
  models = list(selected_models or [])
672
  available = samples.get("models") or []
673
  models = [model for model in models if model in samples.get("images", {})]
@@ -722,9 +835,9 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
722
  cells.append(
723
  f"""
724
  <div class="compare-cell">
725
- <div class="compare-model-label">{escape(model)}</div>
726
  <a href="{image_url}" target="_blank" rel="noopener noreferrer">
727
- <img src="{image_url}" alt="{escape(model)} sample" loading="lazy" />
728
  </a>
729
  </div>
730
  """
@@ -755,38 +868,55 @@ def _note_markdown(note):
755
  return note or ""
756
 
757
 
758
- def _filter_row(datasets, metrics, default_dataset_id, default_metric_id):
 
 
 
 
 
 
 
 
759
  metric_choices = _metric_choices(datasets, metrics, default_dataset_id)
760
  model_choices = _model_choices(datasets, default_dataset_id)
 
 
 
761
  with gr.Row(elem_classes="view-filters"):
762
  dataset_dd = gr.Dropdown(
763
- choices=_dataset_choices(datasets),
764
  value=default_dataset_id,
765
  label="Dataset",
766
  type="value",
767
  scale=2,
768
  min_width=160,
769
  )
770
- metric_dd = gr.Dropdown(
771
- choices=metric_choices,
772
- value=default_metric_id,
773
- label="Metric",
774
- type="value",
775
- scale=2,
776
- min_width=180,
777
- )
 
 
 
 
 
778
  models_dd = gr.Dropdown(
779
  choices=model_choices,
780
  value=[],
781
  multiselect=True,
782
  label="Models",
783
  type="value",
 
784
  scale=3,
785
  min_width=200,
786
  )
787
  title = gr.Markdown(
788
  _title_markdown(
789
- _view_title(datasets, metrics, default_dataset_id, default_metric_id)
790
  ),
791
  elem_classes="view-title",
792
  )
@@ -800,14 +930,17 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
800
  initial = resolve_view(datasets, metrics, default_dataset_id, default_metric_id)
801
  initial_data = initial["data"]
802
  initial_columns = initial["columns"]
803
- initial_score = initial["score_column"]
804
  initial_samples = initial.get("samples")
805
- price_fig, time_fig = _pareto_pair(initial_data, initial_score)
 
 
 
806
 
807
  with gr.Tabs(elem_classes="main-tabs"):
808
  with gr.TabItem("Leaderboards"):
809
  lb_dataset, lb_metric, lb_models, lb_title = _filter_row(
810
- datasets, metrics, default_dataset_id, default_metric_id
811
  )
812
  lb_note = gr.Markdown(_note_markdown(initial.get("note")))
813
  platform_choices = _filter_choices(initial_data, "Platform")
@@ -847,7 +980,10 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
847
  )
848
  ranking = gr.HTML(
849
  _leaderboard_html(
850
- initial_data, initial_columns, [initial_score], initial_score
 
 
 
851
  ),
852
  padding=False,
853
  elem_classes="ranking-table-host",
@@ -855,65 +991,102 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
855
 
856
  with gr.TabItem("Pareto Plots"):
857
  pp_dataset, pp_metric, pp_models, pp_title = _filter_row(
858
- datasets, metrics, default_dataset_id, default_metric_id
859
  )
860
  gr.Markdown(
861
  "<span class='pareto-help'>"
862
  "Green = on the frontier (lower cost or time at the same or better score). "
863
- "Lavender = below the frontier."
 
864
  "</span><br/>"
865
  "<strong class='pareto-help-emphasis'>Hover a point to see which model it is.</strong>"
866
  )
867
- with gr.Row(equal_height=False, elem_classes="pareto-layout"):
868
- with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
869
- gr.Markdown("#### Price vs score")
870
- price_plot = gr.Plot(
871
- value=price_fig,
872
- show_label=False,
873
- elem_classes="pareto-plot",
874
- )
875
- with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
876
- gr.Markdown("#### Min generation time vs score")
877
- time_plot = gr.Plot(
878
- value=time_fig,
879
- show_label=False,
880
- elem_classes="pareto-plot",
 
881
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
882
 
883
  with gr.TabItem("Samples"):
884
- sm_dataset, sm_metric, sm_models, sm_title = _filter_row(
885
- datasets, metrics, default_dataset_id, default_metric_id
886
  )
887
- gr.Markdown(
888
- f"""
889
- <p class="compare-samples-help">
890
- Filter models above (up to <strong>{MAX_COMPARE_MODELS}</strong> are
891
- shown). If none are selected, two defaults appear. Images come from
892
- the public generation URLs for this dataset. Prompts to show chooses
893
- how many shared prompts appear (1–{MAX_COMPARE_PROMPTS}).
894
- </p>
895
- """
896
  )
897
- with gr.Row(equal_height=False, elem_classes="compare-controls"):
898
- prompt_count = gr.Slider(
899
- minimum=1,
900
- maximum=MAX_COMPARE_PROMPTS,
901
- value=DEFAULT_COMPARE_PROMPTS,
902
- step=1,
903
- label="Prompts to show",
904
- container=False,
905
- show_reset_button=False,
906
- scale=1,
907
- min_width=180,
908
- elem_classes="compare-prompt-count",
909
- )
910
- shuffle_button = gr.Button(
911
- "Shuffle prompts",
912
- variant="primary",
913
- scale=0,
914
- min_width=140,
915
- elem_classes="compare-shuffle",
916
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
917
  gallery = gr.HTML(
918
  value=_samples_html(
919
  initial_samples, [], DEFAULT_COMPARE_PROMPTS, seed=0
@@ -925,33 +1098,49 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
925
  with gr.TabItem("About"):
926
  render_about()
927
 
928
- def _synced_filters(dataset_id, metric_id, models):
929
- metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
 
 
 
930
  model_choices = _model_choices(datasets, dataset_id)
931
- models = [model for model in (models or []) if model in model_choices]
 
932
  metric_choices = _metric_choices(datasets, metrics, dataset_id)
933
  title = _title_markdown(
934
  _view_title(datasets, metrics, dataset_id, metric_id)
935
  )
936
  dataset_update = gr.update(value=dataset_id)
937
- metric_update = gr.update(choices=metric_choices, value=metric_id)
 
 
938
  models_update = gr.update(choices=model_choices, value=models)
 
 
 
 
 
 
 
 
 
 
939
  return (
940
  dataset_id,
941
  metric_id,
942
  models,
 
943
  dataset_update,
944
  dataset_update,
945
- dataset_update,
946
  metric_update,
947
  metric_update,
948
- metric_update,
949
- models_update,
950
  models_update,
951
  models_update,
 
952
  title,
953
  title,
954
- title,
955
  )
956
 
957
  def _leaderboard_extras(data, platform_value, owner_value, optimized_value):
@@ -1009,28 +1198,38 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1009
  optimized_value or [],
1010
  models=models,
1011
  )
 
 
 
1012
  ranking_html = _leaderboard_html(
1013
  filtered,
1014
  view["columns"],
1015
- [view["score_column"]],
1016
- view["score_column"],
1017
  )
1018
- pareto_data = _filter_leaderboard(
1019
- data, "", [], [], [], models=models
1020
- )
1021
- next_price, next_time = _pareto_pair(pareto_data, view["score_column"])
 
 
 
 
 
 
 
1022
  samples_html = _samples_html(
1023
- view.get("samples"),
1024
- models,
1025
  int(num_prompts or DEFAULT_COMPARE_PROMPTS),
1026
  int(seed or 0),
1027
  )
1028
  return (
1029
  _note_markdown(view.get("note")),
1030
  ranking_html,
1031
- next_price,
1032
- next_time,
1033
  samples_html,
 
1034
  )
1035
 
1036
  def on_dataset(
@@ -1044,7 +1243,9 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1044
  num_prompts,
1045
  seed,
1046
  ):
1047
- synced = _synced_filters(dataset_id, metric_id, models)
 
 
1048
  dataset_id, metric_id, models = synced[:3]
1049
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
1050
  extras = _leaderboard_extras(
@@ -1061,7 +1262,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1061
  num_prompts,
1062
  seed,
1063
  )
1064
- return (*synced[3:], extras[0], extras[1], extras[2], *views)
1065
 
1066
  def on_metric(
1067
  dataset_id,
@@ -1075,10 +1276,21 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1075
  seed,
1076
  ):
1077
  metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
 
1078
  title = _title_markdown(
1079
  _view_title(datasets, metrics, dataset_id, metric_id)
1080
  )
1081
- metric_update = gr.update(value=metric_id)
 
 
 
 
 
 
 
 
 
 
1082
  views = _views(
1083
  dataset_id,
1084
  metric_id,
@@ -1090,7 +1302,14 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1090
  num_prompts,
1091
  seed,
1092
  )
1093
- return (metric_update, metric_update, metric_update, title, title, title, *views)
 
 
 
 
 
 
 
1094
 
1095
  def on_models(
1096
  dataset_id,
@@ -1104,8 +1323,19 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1104
  seed,
1105
  ):
1106
  model_choices = _model_choices(datasets, dataset_id)
1107
- models = [model for model in (models or []) if model in model_choices]
1108
- models_update = gr.update(value=models)
 
 
 
 
 
 
 
 
 
 
 
1109
  views = _views(
1110
  dataset_id,
1111
  metric_id,
@@ -1117,7 +1347,12 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1117
  num_prompts,
1118
  seed,
1119
  )
1120
- return (models_update, models_update, models_update, *views)
 
 
 
 
 
1121
 
1122
  def on_leaderboard_filters(
1123
  dataset_id,
@@ -1137,11 +1372,14 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1137
  optimized_value or [],
1138
  models=models,
1139
  )
 
 
 
1140
  return _leaderboard_html(
1141
  filtered,
1142
  view["columns"],
1143
- [view["score_column"]],
1144
- view["score_column"],
1145
  )
1146
 
1147
  def on_samples_controls(dataset_id, models, num_prompts, seed):
@@ -1163,23 +1401,19 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1163
  next_seed,
1164
  )
1165
 
1166
- dataset_inputs = [
1167
- lb_metric,
1168
- lb_models,
1169
- search,
1170
- platform,
1171
- owner,
1172
- optimized,
1173
- prompt_count,
1174
- seed_state,
1175
  ]
 
1176
  dataset_outputs = [
1177
  lb_dataset,
1178
  pp_dataset,
1179
  sm_dataset,
1180
  lb_metric,
1181
  pp_metric,
1182
- sm_metric,
1183
  lb_models,
1184
  pp_models,
1185
  sm_models,
@@ -1191,37 +1425,53 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1191
  optimized,
1192
  lb_note,
1193
  ranking,
1194
- price_plot,
1195
- time_plot,
1196
  gallery,
 
1197
  ]
1198
- for dataset_dd in (lb_dataset, pp_dataset, sm_dataset):
 
 
 
 
1199
  dataset_dd.change(
1200
  on_dataset,
1201
- inputs=[dataset_dd, *dataset_inputs],
 
 
 
 
 
 
 
 
 
 
1202
  outputs=dataset_outputs,
1203
  )
1204
 
1205
  metric_outputs = [
1206
  lb_metric,
1207
  pp_metric,
1208
- sm_metric,
1209
  lb_title,
1210
  pp_title,
1211
  sm_title,
1212
  lb_note,
1213
  ranking,
1214
- price_plot,
1215
- time_plot,
1216
  gallery,
 
1217
  ]
1218
- for metric_dd in (lb_metric, pp_metric, sm_metric):
 
 
 
1219
  metric_dd.change(
1220
  on_metric,
1221
  inputs=[
1222
- lb_dataset,
1223
  metric_dd,
1224
- lb_models,
1225
  search,
1226
  platform,
1227
  owner,
@@ -1238,16 +1488,20 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1238
  sm_models,
1239
  lb_note,
1240
  ranking,
1241
- price_plot,
1242
- time_plot,
1243
  gallery,
 
1244
  ]
1245
- for models_dd in (lb_models, pp_models, sm_models):
 
 
 
 
1246
  models_dd.change(
1247
  on_models,
1248
  inputs=[
1249
- lb_dataset,
1250
- lb_metric,
1251
  models_dd,
1252
  search,
1253
  platform,
@@ -1285,7 +1539,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1285
  outputs=[seed_state, gallery],
1286
  )
1287
 
1288
-
1289
  def render_about():
1290
  with gr.Row(elem_classes="about-layout", equal_height=False):
1291
  with gr.Column(scale=1, min_width=340, elem_classes="about-col"):
 
7
  import pandas as pd
8
  import plotly.graph_objects as go
9
 
10
+ from model_display import display_model_name
11
+
12
  _MASCOT_PATH = Path(__file__).resolve().parent / "pruna-mascot.png"
13
  _MASCOT_DATA_URI = (
14
  "data:image/png;base64,"
 
21
  MAX_COMPARE_MODELS = 4
22
  DEFAULT_COMPARE_PROMPTS = 3
23
  MAX_COMPARE_PROMPTS = 8
24
+ MAX_PARETO_METRICS = 8
25
 
26
  ABOUT_OVERVIEW_CONTENT = """
27
  # About P-Bench
 
46
  2. It generates one image per prompt when the run succeeds. Not every model
47
  has every prompt or every metric.
48
  3. Quality is scored automatically (OneIG alignment, P-Judger) and, where
49
+ available, by human preference (Datapoint Elo, Rapidata Elo).
 
50
  4. Price per image and generation time are joined from the evaluation table.
51
 
52
  ## Current datasets
 
61
  general object prompts (100 prompts each). This is the alignment slice of
62
  OneIG, not the full suite. Alignment Overall is the mean of the category
63
  scores that exist for that row.
 
 
 
 
64
  """
65
 
66
  ABOUT_DETAILS_CONTENT = """
 
77
  - **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.
78
  Rapidata rejects prompts over 400 characters, so this Elo is on a subset
79
  of each suite (see Setup). Rapidata is not a dataset.
 
 
80
  - **Generation time**: median and minimum generation time in seconds, as
81
  reported in the evaluation table. This is not a p95, and we do not state
82
  warm vs cold or concurrent load.
 
182
  return items[0] if items else None
183
 
184
 
185
+ def _dataset_choices(datasets, *, require_samples=False):
186
+ return [
187
+ (dataset["name"], dataset["id"])
188
+ for dataset in datasets
189
+ if not require_samples or dataset.get("samples")
190
+ ]
191
+
192
+
193
+ def _coerce_sample_dataset(datasets, dataset_id):
194
+ """Prefer datasets that have compare-sample artifacts."""
195
+ dataset = _item(datasets, dataset_id)
196
+ if dataset and dataset.get("samples"):
197
+ return dataset["id"]
198
+ for candidate in datasets:
199
+ if candidate.get("samples"):
200
+ return candidate["id"]
201
+ return dataset_id
202
 
203
 
204
  def _metric_choices(datasets, metrics, dataset_id):
 
216
 
217
 
218
  def _coerce_metric(datasets, metrics, dataset_id, metric_id):
219
+ """Keep a selected metric only if it is valid for this dataset."""
220
+ if isinstance(metric_id, (list, tuple)):
221
+ metric_id = metric_id[0] if metric_id else None
222
+ if metric_id is None or metric_id == "":
223
+ return None
224
  choices = _metric_choices(datasets, metrics, dataset_id)
225
  ids = [choice[1] for choice in choices]
226
  if metric_id in ids:
227
  return metric_id
228
+ return None
229
+
230
+
231
+ def _metric_dropdown_value(metric_id):
232
+ """Gradio multiselect value for an optional single metric."""
233
+ return [metric_id] if metric_id else []
234
 
235
 
236
  def _model_choices(datasets, dataset_id):
 
238
  data = dataset.get("data") if dataset else None
239
  if data is None or "Model" not in getattr(data, "columns", []):
240
  return []
241
+ models = data["Model"].dropna().astype(str).unique().tolist()
242
+ # (label, value) so the UI shows the shared name but filters on the raw id.
243
+ return sorted(
244
+ ((display_model_name(model), model) for model in models),
245
+ key=lambda item: item[0].casefold(),
246
+ )
247
+
248
+
249
+ def _model_choice_values(choices):
250
+ return [choice[1] if isinstance(choice, tuple) else choice for choice in choices]
251
+
252
+
253
+ def _metric_columns(datasets, metrics, dataset_id):
254
+ dataset = _item(datasets, dataset_id)
255
+ if not dataset:
256
+ return []
257
+ data = dataset.get("data")
258
+ columns = getattr(data, "columns", [])
259
+ allowed = set(dataset.get("metric_ids") or [])
260
+ return [
261
+ metric["column"]
262
+ for metric in metrics
263
+ if metric["id"] in allowed and metric["column"] in columns
264
+ ]
265
 
266
 
267
  def _view_title(datasets, metrics, dataset_id, metric_id):
268
  dataset = _item(datasets, dataset_id)
 
269
  dataset_name = dataset["name"] if dataset else "Dataset"
270
+ metric = _item(metrics, metric_id) if metric_id else None
271
+ if not metric:
272
+ return dataset_name
273
+ return f"{dataset_name} | {metric['name']}"
274
+
275
+
276
+ _LEADERBOARD_META_COLUMNS = [
277
+ "Median Generation Time (s)",
278
+ "Min Generation Time (s)",
279
+ "Price / Image (USD)",
280
+ "Evaluation Date (UTC)",
281
+ "Date",
282
+ ]
283
 
284
 
285
  def _columns_for_metric(dataset, metric_column):
286
+ """When a metric is selected, show identity + that score + time/price."""
287
+ available = list(getattr(dataset.get("data"), "columns", [])) or list(
288
+ dataset.get("columns") or []
289
+ )
290
+ identity = [
291
+ column
292
+ for column in ["Model", "Platform", "Endpoint Owner", "Optimized"]
293
+ if column in available
294
+ ]
295
+ meta = [column for column in _LEADERBOARD_META_COLUMNS if column in available]
296
+ if metric_column and metric_column in available:
297
+ return [*identity, metric_column, *meta]
298
+ return [column for column in (dataset.get("columns") or available) if column != "URL"]
299
 
300
 
301
  def resolve_view(datasets, metrics, dataset_id, metric_id):
302
  dataset = _item(datasets, dataset_id)
303
+ if not dataset:
 
 
304
  return None
305
+ metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
306
+ metric = _item(metrics, metric_id) if metric_id else None
307
+ score_column = metric["column"] if metric else None
308
+ if score_column:
309
+ columns = _columns_for_metric(dataset, score_column)
310
+ score_columns = [score_column]
311
+ else:
312
+ columns = [
313
+ column
314
+ for column in (dataset.get("columns") or [])
315
+ if column != "URL"
316
+ ]
317
+ score_columns = _metric_columns(datasets, metrics, dataset_id)
318
  return {
319
  "dataset": dataset,
320
  "metric": metric,
321
+ "metric_id": metric_id,
322
+ "title": _view_title(datasets, metrics, dataset["id"], metric_id),
323
  "data": dataset["data"],
324
+ "columns": columns,
325
+ "score_column": score_column,
326
+ "score_columns": score_columns,
327
  "samples": dataset.get("samples"),
328
  "note": dataset.get("note"),
329
  }
 
343
  except (TypeError, ValueError):
344
  return escape(str(value))
345
  if label in {"model", "platform", "endpoint owner", "optimized"}:
346
+ if label == "model":
347
+ return escape(display_model_name(value))
348
  return escape(str(value))
349
  try:
350
  number = float(value)
 
457
  matches |= filtered[column].astype(str).str.contains(
458
  search_term, case=False, na=False
459
  )
460
+ if "Model" in filtered.columns:
461
+ display_names = filtered["Model"].map(display_model_name).astype(str)
462
+ matches |= display_names.str.contains(
463
+ search_term, case=False, na=False
464
+ )
465
  filtered = filtered[matches]
466
 
467
  for column, values in [
 
545
  "P-Judge Overall": "P-Judger (Pruna)",
546
  "Datapoint Elo": "Datapoint Elo",
547
  "Rapidata Elo": "Rapidata Elo",
548
+ "Arena Elo": "Overall Elo",
549
+ "Arena Branding / Commercial Elo": "Branding / Commercial",
550
+ "Arena 3D Imaging Elo": "3D Imaging",
551
+ "Arena Cartoon / Anime Elo": "Cartoon / Anime",
552
+ "Arena Photorealistic Elo": "Photorealistic",
553
+ "Arena Art Elo": "Art",
554
+ "Arena Portraits Elo": "Portraits",
555
+ "Arena Text Rendering Elo": "Text Rendering",
556
  "Raw Win Rate": "Raw win rate",
557
  "Median Generation Time (s)": "Median generation time",
558
  "Min Generation Time (s)": "Min generation time",
 
629
  scores = scatter[score_column].astype(float).tolist()
630
  on_frontier = _pareto_frontier_mask(x_values, scores)
631
 
632
+ dominated = scatter.loc[[not flag for flag in on_frontier]].copy()
633
+ frontier = scatter.loc[on_frontier].sort_values(x_column).copy()
634
+ if not dominated.empty:
635
+ dominated["Model"] = dominated["Model"].map(display_model_name)
636
+ if not frontier.empty:
637
+ frontier["Model"] = frontier["Model"].map(display_model_name)
638
  hover = (
639
  "<b>%{text}</b><br>"
640
  f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"
 
750
  )
751
 
752
 
753
+ def _pareto_slot_updates(data, score_columns):
754
+ """Updates for a fixed bank of Gradio Plot slots (visible/hidden)."""
755
+ score_columns = [column for column in (score_columns or []) if column]
756
+ updates = []
757
+ for index in range(MAX_PARETO_METRICS):
758
+ if index < len(score_columns):
759
+ score_column = score_columns[index]
760
+ price_fig, time_fig = _pareto_pair(data, score_column)
761
+ updates.extend(
762
+ [
763
+ gr.update(visible=True),
764
+ f"#### {_display_label(score_column)}",
765
+ price_fig,
766
+ time_fig,
767
+ ]
768
+ )
769
+ else:
770
+ updates.extend(
771
+ [
772
+ gr.update(visible=False),
773
+ "",
774
+ None,
775
+ None,
776
+ ]
777
+ )
778
+ return updates
779
+
780
+
781
  def _samples_html(samples, selected_models, num_prompts, seed=0):
782
  if not samples:
783
+ return ""
 
 
 
 
784
  models = list(selected_models or [])
785
  available = samples.get("models") or []
786
  models = [model for model in models if model in samples.get("images", {})]
 
835
  cells.append(
836
  f"""
837
  <div class="compare-cell">
838
+ <div class="compare-model-label">{escape(display_model_name(model))}</div>
839
  <a href="{image_url}" target="_blank" rel="noopener noreferrer">
840
+ <img src="{image_url}" alt="{escape(display_model_name(model))} sample" loading="lazy" />
841
  </a>
842
  </div>
843
  """
 
868
  return note or ""
869
 
870
 
871
+ def _filter_row(
872
+ datasets,
873
+ metrics,
874
+ default_dataset_id,
875
+ default_metric_id=None,
876
+ *,
877
+ require_samples=False,
878
+ include_metric=True,
879
+ ):
880
  metric_choices = _metric_choices(datasets, metrics, default_dataset_id)
881
  model_choices = _model_choices(datasets, default_dataset_id)
882
+ metric_id = _coerce_metric(
883
+ datasets, metrics, default_dataset_id, default_metric_id
884
+ )
885
  with gr.Row(elem_classes="view-filters"):
886
  dataset_dd = gr.Dropdown(
887
+ choices=_dataset_choices(datasets, require_samples=require_samples),
888
  value=default_dataset_id,
889
  label="Dataset",
890
  type="value",
891
  scale=2,
892
  min_width=160,
893
  )
894
+ metric_dd = None
895
+ if include_metric:
896
+ metric_dd = gr.Dropdown(
897
+ choices=metric_choices,
898
+ value=_metric_dropdown_value(metric_id),
899
+ label="Metric",
900
+ type="value",
901
+ multiselect=True,
902
+ max_choices=1,
903
+ allow_custom_value=True,
904
+ scale=2,
905
+ min_width=180,
906
+ )
907
  models_dd = gr.Dropdown(
908
  choices=model_choices,
909
  value=[],
910
  multiselect=True,
911
  label="Models",
912
  type="value",
913
+ allow_custom_value=True,
914
  scale=3,
915
  min_width=200,
916
  )
917
  title = gr.Markdown(
918
  _title_markdown(
919
+ _view_title(datasets, metrics, default_dataset_id, metric_id)
920
  ),
921
  elem_classes="view-title",
922
  )
 
930
  initial = resolve_view(datasets, metrics, default_dataset_id, default_metric_id)
931
  initial_data = initial["data"]
932
  initial_columns = initial["columns"]
933
+ initial_score_columns = initial["score_columns"]
934
  initial_samples = initial.get("samples")
935
+ initial_pareto_updates = _pareto_slot_updates(
936
+ _filter_leaderboard(initial_data, "", [], [], [], models=[]),
937
+ initial_score_columns,
938
+ )
939
 
940
  with gr.Tabs(elem_classes="main-tabs"):
941
  with gr.TabItem("Leaderboards"):
942
  lb_dataset, lb_metric, lb_models, lb_title = _filter_row(
943
+ datasets, metrics, default_dataset_id, None
944
  )
945
  lb_note = gr.Markdown(_note_markdown(initial.get("note")))
946
  platform_choices = _filter_choices(initial_data, "Platform")
 
980
  )
981
  ranking = gr.HTML(
982
  _leaderboard_html(
983
+ initial_data,
984
+ initial_columns,
985
+ initial_score_columns,
986
+ initial_score_columns[0] if initial_score_columns else None,
987
  ),
988
  padding=False,
989
  elem_classes="ranking-table-host",
 
991
 
992
  with gr.TabItem("Pareto Plots"):
993
  pp_dataset, pp_metric, pp_models, pp_title = _filter_row(
994
+ datasets, metrics, default_dataset_id, None
995
  )
996
  gr.Markdown(
997
  "<span class='pareto-help'>"
998
  "Green = on the frontier (lower cost or time at the same or better score). "
999
+ "Lavender = below the frontier. Pick a metric to focus on one pair of plots; "
1000
+ "leave it empty to see every metric."
1001
  "</span><br/>"
1002
  "<strong class='pareto-help-emphasis'>Hover a point to see which model it is.</strong>"
1003
  )
1004
+ pareto_slots = []
1005
+ for slot_index in range(MAX_PARETO_METRICS):
1006
+ offset = slot_index * 4
1007
+ visible = slot_index < len(initial_score_columns)
1008
+ with gr.Column(
1009
+ visible=visible,
1010
+ elem_classes="pareto-metric-block",
1011
+ ) as slot_group:
1012
+ slot_title = gr.Markdown(
1013
+ (
1014
+ f"#### {_display_label(initial_score_columns[slot_index])}"
1015
+ if visible
1016
+ else ""
1017
+ ),
1018
+ elem_classes="pareto-metric-title",
1019
  )
1020
+ with gr.Row(equal_height=False, elem_classes="pareto-layout"):
1021
+ with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
1022
+ gr.Markdown("#### Price vs score")
1023
+ slot_price = gr.Plot(
1024
+ value=(
1025
+ initial_pareto_updates[offset + 2]
1026
+ if visible
1027
+ else None
1028
+ ),
1029
+ show_label=False,
1030
+ elem_classes="pareto-plot",
1031
+ )
1032
+ with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
1033
+ gr.Markdown("#### Min generation time vs score")
1034
+ slot_time = gr.Plot(
1035
+ value=(
1036
+ initial_pareto_updates[offset + 3]
1037
+ if visible
1038
+ else None
1039
+ ),
1040
+ show_label=False,
1041
+ elem_classes="pareto-plot",
1042
+ )
1043
+ pareto_slots.append(
1044
+ (slot_group, slot_title, slot_price, slot_time)
1045
+ )
1046
 
1047
  with gr.TabItem("Samples"):
1048
+ sample_default_dataset_id = _coerce_sample_dataset(
1049
+ datasets, default_dataset_id
1050
  )
1051
+ sm_dataset, _, sm_models, sm_title = _filter_row(
1052
+ datasets,
1053
+ metrics,
1054
+ sample_default_dataset_id,
1055
+ None,
1056
+ require_samples=True,
1057
+ include_metric=False,
 
 
1058
  )
1059
+ with gr.Column(visible=bool(initial_samples)) as samples_panel:
1060
+ gr.Markdown(
1061
+ f"""
1062
+ <p class="compare-samples-help">
1063
+ Filter models above (up to <strong>{MAX_COMPARE_MODELS}</strong> are
1064
+ shown). If none are selected, two defaults appear. Images come from
1065
+ the public generation URLs for this dataset. Prompts to show chooses
1066
+ how many shared prompts appear (1–{MAX_COMPARE_PROMPTS}).
1067
+ </p>
1068
+ """
 
 
 
 
 
 
 
 
 
1069
  )
1070
+ with gr.Row(equal_height=False, elem_classes="compare-controls"):
1071
+ prompt_count = gr.Slider(
1072
+ minimum=1,
1073
+ maximum=MAX_COMPARE_PROMPTS,
1074
+ value=DEFAULT_COMPARE_PROMPTS,
1075
+ step=1,
1076
+ label="Prompts to show",
1077
+ container=False,
1078
+ show_reset_button=False,
1079
+ scale=1,
1080
+ min_width=180,
1081
+ elem_classes="compare-prompt-count",
1082
+ )
1083
+ shuffle_button = gr.Button(
1084
+ "Shuffle prompts",
1085
+ variant="primary",
1086
+ scale=0,
1087
+ min_width=140,
1088
+ elem_classes="compare-shuffle",
1089
+ )
1090
  gallery = gr.HTML(
1091
  value=_samples_html(
1092
  initial_samples, [], DEFAULT_COMPARE_PROMPTS, seed=0
 
1098
  with gr.TabItem("About"):
1099
  render_about()
1100
 
1101
+ def _synced_filters(dataset_id, metric_id, models, *, clear_metric=False):
1102
+ if clear_metric:
1103
+ metric_id = None
1104
+ else:
1105
+ metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1106
  model_choices = _model_choices(datasets, dataset_id)
1107
+ model_values = set(_model_choice_values(model_choices))
1108
+ models = [model for model in (models or []) if model in model_values]
1109
  metric_choices = _metric_choices(datasets, metrics, dataset_id)
1110
  title = _title_markdown(
1111
  _view_title(datasets, metrics, dataset_id, metric_id)
1112
  )
1113
  dataset_update = gr.update(value=dataset_id)
1114
+ metric_update = gr.update(
1115
+ choices=metric_choices, value=_metric_dropdown_value(metric_id)
1116
+ )
1117
  models_update = gr.update(choices=model_choices, value=models)
1118
+
1119
+ sample_dataset_id = _coerce_sample_dataset(datasets, dataset_id)
1120
+ sample_model_choices = _model_choices(datasets, sample_dataset_id)
1121
+ sample_model_values = set(_model_choice_values(sample_model_choices))
1122
+ sample_models = [
1123
+ model for model in (models or []) if model in sample_model_values
1124
+ ]
1125
+ sample_title = _title_markdown(
1126
+ _view_title(datasets, metrics, sample_dataset_id, None)
1127
+ )
1128
  return (
1129
  dataset_id,
1130
  metric_id,
1131
  models,
1132
+ sample_dataset_id,
1133
  dataset_update,
1134
  dataset_update,
1135
+ gr.update(value=sample_dataset_id),
1136
  metric_update,
1137
  metric_update,
 
 
1138
  models_update,
1139
  models_update,
1140
+ gr.update(choices=sample_model_choices, value=sample_models),
1141
  title,
1142
  title,
1143
+ sample_title,
1144
  )
1145
 
1146
  def _leaderboard_extras(data, platform_value, owner_value, optimized_value):
 
1198
  optimized_value or [],
1199
  models=models,
1200
  )
1201
+ sort_column = view["score_column"] or (
1202
+ view["score_columns"][0] if view["score_columns"] else None
1203
+ )
1204
  ranking_html = _leaderboard_html(
1205
  filtered,
1206
  view["columns"],
1207
+ view["score_columns"],
1208
+ sort_column,
1209
  )
1210
+ pareto_data = _filter_leaderboard(data, "", [], [], [], models=models)
1211
+ pareto_updates = _pareto_slot_updates(pareto_data, view["score_columns"])
1212
+ sample_dataset_id = _coerce_sample_dataset(datasets, dataset_id)
1213
+ sample_view = resolve_view(datasets, metrics, sample_dataset_id, None)
1214
+ samples = sample_view.get("samples") if sample_view else None
1215
+ sample_models = [
1216
+ model
1217
+ for model in (models or [])
1218
+ if model
1219
+ in _model_choice_values(_model_choices(datasets, sample_dataset_id))
1220
+ ]
1221
  samples_html = _samples_html(
1222
+ samples,
1223
+ sample_models,
1224
  int(num_prompts or DEFAULT_COMPARE_PROMPTS),
1225
  int(seed or 0),
1226
  )
1227
  return (
1228
  _note_markdown(view.get("note")),
1229
  ranking_html,
1230
+ *pareto_updates,
 
1231
  samples_html,
1232
+ gr.update(visible=bool(samples)),
1233
  )
1234
 
1235
  def on_dataset(
 
1243
  num_prompts,
1244
  seed,
1245
  ):
1246
+ synced = _synced_filters(
1247
+ dataset_id, metric_id, models, clear_metric=True
1248
+ )
1249
  dataset_id, metric_id, models = synced[:3]
1250
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
1251
  extras = _leaderboard_extras(
 
1262
  num_prompts,
1263
  seed,
1264
  )
1265
+ return (*synced[4:], extras[0], extras[1], extras[2], *views)
1266
 
1267
  def on_metric(
1268
  dataset_id,
 
1276
  seed,
1277
  ):
1278
  metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1279
+ metric_choices = _metric_choices(datasets, metrics, dataset_id)
1280
  title = _title_markdown(
1281
  _view_title(datasets, metrics, dataset_id, metric_id)
1282
  )
1283
+ metric_update = gr.update(
1284
+ choices=metric_choices, value=_metric_dropdown_value(metric_id)
1285
+ )
1286
+ sample_title = _title_markdown(
1287
+ _view_title(
1288
+ datasets,
1289
+ metrics,
1290
+ _coerce_sample_dataset(datasets, dataset_id),
1291
+ None,
1292
+ )
1293
+ )
1294
  views = _views(
1295
  dataset_id,
1296
  metric_id,
 
1302
  num_prompts,
1303
  seed,
1304
  )
1305
+ return (
1306
+ metric_update,
1307
+ metric_update,
1308
+ title,
1309
+ title,
1310
+ sample_title,
1311
+ *views,
1312
+ )
1313
 
1314
  def on_models(
1315
  dataset_id,
 
1323
  seed,
1324
  ):
1325
  model_choices = _model_choices(datasets, dataset_id)
1326
+ model_values = set(_model_choice_values(model_choices))
1327
+ models = [model for model in (models or []) if model in model_values]
1328
+ models_update = gr.update(choices=model_choices, value=models)
1329
+
1330
+ sample_dataset_id = _coerce_sample_dataset(datasets, dataset_id)
1331
+ sample_model_choices = _model_choices(datasets, sample_dataset_id)
1332
+ sample_model_values = set(_model_choice_values(sample_model_choices))
1333
+ sample_models = [
1334
+ model for model in (models or []) if model in sample_model_values
1335
+ ]
1336
+ sample_models_update = gr.update(
1337
+ choices=sample_model_choices, value=sample_models
1338
+ )
1339
  views = _views(
1340
  dataset_id,
1341
  metric_id,
 
1347
  num_prompts,
1348
  seed,
1349
  )
1350
+ return (
1351
+ models_update,
1352
+ models_update,
1353
+ sample_models_update,
1354
+ *views,
1355
+ )
1356
 
1357
  def on_leaderboard_filters(
1358
  dataset_id,
 
1372
  optimized_value or [],
1373
  models=models,
1374
  )
1375
+ sort_column = view["score_column"] or (
1376
+ view["score_columns"][0] if view["score_columns"] else None
1377
+ )
1378
  return _leaderboard_html(
1379
  filtered,
1380
  view["columns"],
1381
+ view["score_columns"],
1382
+ sort_column,
1383
  )
1384
 
1385
  def on_samples_controls(dataset_id, models, num_prompts, seed):
 
1401
  next_seed,
1402
  )
1403
 
1404
+ no_metric = gr.State(None)
1405
+ pareto_outputs = [
1406
+ component
1407
+ for slot_group, slot_title, slot_price, slot_time in pareto_slots
1408
+ for component in (slot_group, slot_title, slot_price, slot_time)
 
 
 
 
1409
  ]
1410
+
1411
  dataset_outputs = [
1412
  lb_dataset,
1413
  pp_dataset,
1414
  sm_dataset,
1415
  lb_metric,
1416
  pp_metric,
 
1417
  lb_models,
1418
  pp_models,
1419
  sm_models,
 
1425
  optimized,
1426
  lb_note,
1427
  ranking,
1428
+ *pareto_outputs,
 
1429
  gallery,
1430
+ samples_panel,
1431
  ]
1432
+ for dataset_dd, metric_dd, models_dd in (
1433
+ (lb_dataset, lb_metric, lb_models),
1434
+ (pp_dataset, pp_metric, pp_models),
1435
+ (sm_dataset, no_metric, sm_models),
1436
+ ):
1437
  dataset_dd.change(
1438
  on_dataset,
1439
+ inputs=[
1440
+ dataset_dd,
1441
+ metric_dd,
1442
+ models_dd,
1443
+ search,
1444
+ platform,
1445
+ owner,
1446
+ optimized,
1447
+ prompt_count,
1448
+ seed_state,
1449
+ ],
1450
  outputs=dataset_outputs,
1451
  )
1452
 
1453
  metric_outputs = [
1454
  lb_metric,
1455
  pp_metric,
 
1456
  lb_title,
1457
  pp_title,
1458
  sm_title,
1459
  lb_note,
1460
  ranking,
1461
+ *pareto_outputs,
 
1462
  gallery,
1463
+ samples_panel,
1464
  ]
1465
+ for dataset_dd, metric_dd, models_dd in (
1466
+ (lb_dataset, lb_metric, lb_models),
1467
+ (pp_dataset, pp_metric, pp_models),
1468
+ ):
1469
  metric_dd.change(
1470
  on_metric,
1471
  inputs=[
1472
+ dataset_dd,
1473
  metric_dd,
1474
+ models_dd,
1475
  search,
1476
  platform,
1477
  owner,
 
1488
  sm_models,
1489
  lb_note,
1490
  ranking,
1491
+ *pareto_outputs,
 
1492
  gallery,
1493
+ samples_panel,
1494
  ]
1495
+ for dataset_dd, metric_dd, models_dd in (
1496
+ (lb_dataset, lb_metric, lb_models),
1497
+ (pp_dataset, pp_metric, pp_models),
1498
+ (sm_dataset, no_metric, sm_models),
1499
+ ):
1500
  models_dd.change(
1501
  on_models,
1502
  inputs=[
1503
+ dataset_dd,
1504
+ metric_dd,
1505
  models_dd,
1506
  search,
1507
  platform,
 
1539
  outputs=[seed_state, gallery],
1540
  )
1541
 
 
1542
  def render_about():
1543
  with gr.Row(elem_classes="about-layout", equal_height=False):
1544
  with gr.Column(scale=1, min_width=340, elem_classes="about-col"):