Spaces:
Running
Running
feat: add AA and Arenaai results, fix some ui features
Browse files- app.py +179 -18
- data/arena_ai_text_to_image_leaderboard.csv +77 -0
- data/artificial_analysis_text_to_image_leaderboard.csv +146 -0
- model_display.py +251 -0
- ui.py +407 -154
app.py
CHANGED
|
@@ -2023,6 +2023,29 @@ footer, .footer { display: none !important; }
|
|
| 2023 |
color: var(--pruna-text-primary) !important;
|
| 2024 |
font-size: 1rem;
|
| 2025 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2026 |
|
| 2027 |
.options,
|
| 2028 |
ul.options,
|
|
@@ -2271,6 +2294,72 @@ def load_oneig_dataframe(path):
|
|
| 2271 |
return df
|
| 2272 |
|
| 2273 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2274 |
def load_qwen_combined_dataframe(path):
|
| 2275 |
"""Load the combined Qwen Image Bench metrics table."""
|
| 2276 |
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
|
|
@@ -2288,7 +2377,6 @@ def load_qwen_combined_dataframe(path):
|
|
| 2288 |
"P-Judge Overall",
|
| 2289 |
"Rapidata Elo",
|
| 2290 |
"Datapoint Elo",
|
| 2291 |
-
"Benchmark.ai Elo",
|
| 2292 |
]:
|
| 2293 |
if column in df.columns:
|
| 2294 |
df[column] = pd.to_numeric(df[column], errors="coerce")
|
|
@@ -2369,8 +2457,18 @@ qwen_path = _resolve_data_path(
|
|
| 2369 |
data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 2370 |
space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 2371 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2372 |
|
| 2373 |
qwen_df = load_qwen_combined_dataframe(qwen_path)
|
|
|
|
|
|
|
| 2374 |
qwen_display_columns = [
|
| 2375 |
col
|
| 2376 |
for col in [
|
|
@@ -2389,12 +2487,19 @@ aa_display_columns = [
|
|
| 2389 |
col
|
| 2390 |
for col in [
|
| 2391 |
"Model",
|
| 2392 |
-
"
|
| 2393 |
-
"Median Generation Time (s)",
|
| 2394 |
-
"Min Generation Time (s)",
|
| 2395 |
"Price / Image (USD)",
|
| 2396 |
]
|
| 2397 |
-
if col in
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2398 |
]
|
| 2399 |
|
| 2400 |
oneig_samples = load_sample_comparison_data(oneig_combined_dir)
|
|
@@ -2439,7 +2544,47 @@ metrics = [
|
|
| 2439 |
{
|
| 2440 |
"id": "aa_elo",
|
| 2441 |
"name": "Artificial Analysis ELO Metric",
|
| 2442 |
-
"column": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2443 |
},
|
| 2444 |
]
|
| 2445 |
|
|
@@ -2467,7 +2612,20 @@ oneig_metric_ids = _metric_ids_for(
|
|
| 2467 |
"datapoint_elo_object",
|
| 2468 |
],
|
| 2469 |
)
|
| 2470 |
-
aa_metric_ids = _metric_ids_for(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2471 |
|
| 2472 |
datasets = [
|
| 2473 |
{
|
|
@@ -2496,14 +2654,21 @@ datasets = [
|
|
| 2496 |
},
|
| 2497 |
{
|
| 2498 |
"id": "artificial_analysis",
|
| 2499 |
-
"name": "Artificial Analysis
|
| 2500 |
-
"data":
|
| 2501 |
"columns": aa_display_columns,
|
| 2502 |
"metric_ids": aa_metric_ids,
|
| 2503 |
-
"note":
|
| 2504 |
-
|
| 2505 |
-
|
| 2506 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2507 |
},
|
| 2508 |
]
|
| 2509 |
datasets = [dataset for dataset in datasets if dataset["metric_ids"]]
|
|
@@ -2512,11 +2677,7 @@ DEFAULT_DATASET_ID = next(
|
|
| 2512 |
(dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
|
| 2513 |
datasets[0]["id"] if datasets else None,
|
| 2514 |
)
|
| 2515 |
-
DEFAULT_METRIC_ID =
|
| 2516 |
-
"datapoint_elo"
|
| 2517 |
-
if DEFAULT_DATASET_ID == "qwen" and "datapoint_elo" in qwen_metric_ids
|
| 2518 |
-
else (datasets[0]["metric_ids"][0] if datasets else None)
|
| 2519 |
-
)
|
| 2520 |
|
| 2521 |
|
| 2522 |
custom_head = """
|
|
|
|
| 2023 |
color: var(--pruna-text-primary) !important;
|
| 2024 |
font-size: 1rem;
|
| 2025 |
}
|
| 2026 |
+
.pareto-panel-host {
|
| 2027 |
+
display: flex;
|
| 2028 |
+
flex-direction: column;
|
| 2029 |
+
gap: 1.5rem;
|
| 2030 |
+
}
|
| 2031 |
+
.pareto-metric-block {
|
| 2032 |
+
padding-bottom: 0.5rem;
|
| 2033 |
+
border-bottom: 1px solid var(--pruna-hairline);
|
| 2034 |
+
}
|
| 2035 |
+
.pareto-metric-block:last-child {
|
| 2036 |
+
border-bottom: none;
|
| 2037 |
+
}
|
| 2038 |
+
.pareto-metric-title {
|
| 2039 |
+
margin: 0 0 0.75rem;
|
| 2040 |
+
color: var(--pruna-text-primary);
|
| 2041 |
+
font-size: 1.05rem;
|
| 2042 |
+
font-weight: 600;
|
| 2043 |
+
}
|
| 2044 |
+
.pareto-subhead {
|
| 2045 |
+
margin: 0 0 0.35rem;
|
| 2046 |
+
color: var(--pruna-text-muted);
|
| 2047 |
+
font-size: 0.9rem;
|
| 2048 |
+
}
|
| 2049 |
|
| 2050 |
.options,
|
| 2051 |
ul.options,
|
|
|
|
| 2294 |
return df
|
| 2295 |
|
| 2296 |
|
| 2297 |
+
def load_artificial_analysis_dataframe(path):
|
| 2298 |
+
"""Load the Artificial Analysis text-to-image leaderboard."""
|
| 2299 |
+
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
|
| 2300 |
+
df = df.rename(
|
| 2301 |
+
columns={
|
| 2302 |
+
"model": "Model",
|
| 2303 |
+
"elo": "Artificial Analysis Elo",
|
| 2304 |
+
"price_per_image_usd": "Price / Image (USD)",
|
| 2305 |
+
}
|
| 2306 |
+
)
|
| 2307 |
+
df["Model"] = df["Model"].astype(str).str.strip()
|
| 2308 |
+
for column in ["Artificial Analysis Elo", "Price / Image (USD)"]:
|
| 2309 |
+
if column in df.columns:
|
| 2310 |
+
df[column] = pd.to_numeric(df[column], errors="coerce")
|
| 2311 |
+
if "Artificial Analysis Elo" in df.columns:
|
| 2312 |
+
df = df.sort_values(
|
| 2313 |
+
by="Artificial Analysis Elo", ascending=False, na_position="last"
|
| 2314 |
+
)
|
| 2315 |
+
numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
|
| 2316 |
+
for col in numeric_cols:
|
| 2317 |
+
df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
|
| 2318 |
+
return df.reset_index(drop=True)
|
| 2319 |
+
|
| 2320 |
+
|
| 2321 |
+
ARENA_CATEGORY_COLUMNS = {
|
| 2322 |
+
"product_branding_commercial_design_elo": "Arena Branding / Commercial Elo",
|
| 2323 |
+
"3d_imaging_and_modeling_elo": "Arena 3D Imaging Elo",
|
| 2324 |
+
"cartoon_anime_and_fantasy_elo": "Arena Cartoon / Anime Elo",
|
| 2325 |
+
"photorealistic_and_cinematic_imagery_elo": "Arena Photorealistic Elo",
|
| 2326 |
+
"art_elo": "Arena Art Elo",
|
| 2327 |
+
"portraits_elo": "Arena Portraits Elo",
|
| 2328 |
+
"text_rendering_elo": "Arena Text Rendering Elo",
|
| 2329 |
+
}
|
| 2330 |
+
|
| 2331 |
+
|
| 2332 |
+
def load_arena_ai_dataframe(path):
|
| 2333 |
+
"""Load the Arena AI text-to-image leaderboard (overall + category Elos)."""
|
| 2334 |
+
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
|
| 2335 |
+
rename_map = {"model": "Model", "elo": "Arena Elo"}
|
| 2336 |
+
rename_map.update(
|
| 2337 |
+
{
|
| 2338 |
+
source: label
|
| 2339 |
+
for source, label in ARENA_CATEGORY_COLUMNS.items()
|
| 2340 |
+
if source in df.columns
|
| 2341 |
+
}
|
| 2342 |
+
)
|
| 2343 |
+
df = df.rename(columns=rename_map)
|
| 2344 |
+
df["Model"] = df["Model"].astype(str).str.strip()
|
| 2345 |
+
score_columns = [
|
| 2346 |
+
column
|
| 2347 |
+
for column in ["Arena Elo", *ARENA_CATEGORY_COLUMNS.values()]
|
| 2348 |
+
if column in df.columns
|
| 2349 |
+
]
|
| 2350 |
+
for column in score_columns:
|
| 2351 |
+
df[column] = pd.to_numeric(df[column], errors="coerce")
|
| 2352 |
+
if "Arena Elo" in df.columns:
|
| 2353 |
+
df = df.sort_values(by="Arena Elo", ascending=False, na_position="last")
|
| 2354 |
+
numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
|
| 2355 |
+
for col in numeric_cols:
|
| 2356 |
+
df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
|
| 2357 |
+
ordered = ["Model", *score_columns]
|
| 2358 |
+
return df[[column for column in ordered if column in df.columns]].reset_index(
|
| 2359 |
+
drop=True
|
| 2360 |
+
)
|
| 2361 |
+
|
| 2362 |
+
|
| 2363 |
def load_qwen_combined_dataframe(path):
|
| 2364 |
"""Load the combined Qwen Image Bench metrics table."""
|
| 2365 |
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
|
|
|
|
| 2377 |
"P-Judge Overall",
|
| 2378 |
"Rapidata Elo",
|
| 2379 |
"Datapoint Elo",
|
|
|
|
| 2380 |
]:
|
| 2381 |
if column in df.columns:
|
| 2382 |
df[column] = pd.to_numeric(df[column], errors="coerce")
|
|
|
|
| 2457 |
data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 2458 |
space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 2459 |
)
|
| 2460 |
+
aa_path = _resolve_data_path(
|
| 2461 |
+
data_dir / "artificial_analysis_text_to_image_leaderboard.csv",
|
| 2462 |
+
space_root.parent / "artificial_analysis_text_to_image_leaderboard.csv",
|
| 2463 |
+
)
|
| 2464 |
+
arena_path = _resolve_data_path(
|
| 2465 |
+
data_dir / "arena_ai_text_to_image_leaderboard.csv",
|
| 2466 |
+
space_root.parent / "arena_ai_text_to_image_leaderboard.csv",
|
| 2467 |
+
)
|
| 2468 |
|
| 2469 |
qwen_df = load_qwen_combined_dataframe(qwen_path)
|
| 2470 |
+
aa_df = load_artificial_analysis_dataframe(aa_path)
|
| 2471 |
+
arena_df = load_arena_ai_dataframe(arena_path)
|
| 2472 |
qwen_display_columns = [
|
| 2473 |
col
|
| 2474 |
for col in [
|
|
|
|
| 2487 |
col
|
| 2488 |
for col in [
|
| 2489 |
"Model",
|
| 2490 |
+
"Artificial Analysis Elo",
|
|
|
|
|
|
|
| 2491 |
"Price / Image (USD)",
|
| 2492 |
]
|
| 2493 |
+
if col in aa_df.columns
|
| 2494 |
+
]
|
| 2495 |
+
arena_display_columns = [
|
| 2496 |
+
col
|
| 2497 |
+
for col in [
|
| 2498 |
+
"Model",
|
| 2499 |
+
"Arena Elo",
|
| 2500 |
+
*ARENA_CATEGORY_COLUMNS.values(),
|
| 2501 |
+
]
|
| 2502 |
+
if col in arena_df.columns
|
| 2503 |
]
|
| 2504 |
|
| 2505 |
oneig_samples = load_sample_comparison_data(oneig_combined_dir)
|
|
|
|
| 2544 |
{
|
| 2545 |
"id": "aa_elo",
|
| 2546 |
"name": "Artificial Analysis ELO Metric",
|
| 2547 |
+
"column": "Artificial Analysis Elo",
|
| 2548 |
+
},
|
| 2549 |
+
{
|
| 2550 |
+
"id": "arena_elo",
|
| 2551 |
+
"name": "Arena ELO - Overall Metric",
|
| 2552 |
+
"column": "Arena Elo",
|
| 2553 |
+
},
|
| 2554 |
+
{
|
| 2555 |
+
"id": "arena_branding",
|
| 2556 |
+
"name": "Arena ELO - Branding / Commercial Metric",
|
| 2557 |
+
"column": "Arena Branding / Commercial Elo",
|
| 2558 |
+
},
|
| 2559 |
+
{
|
| 2560 |
+
"id": "arena_3d",
|
| 2561 |
+
"name": "Arena ELO - 3D Imaging Metric",
|
| 2562 |
+
"column": "Arena 3D Imaging Elo",
|
| 2563 |
+
},
|
| 2564 |
+
{
|
| 2565 |
+
"id": "arena_cartoon",
|
| 2566 |
+
"name": "Arena ELO - Cartoon / Anime Metric",
|
| 2567 |
+
"column": "Arena Cartoon / Anime Elo",
|
| 2568 |
+
},
|
| 2569 |
+
{
|
| 2570 |
+
"id": "arena_photo",
|
| 2571 |
+
"name": "Arena ELO - Photorealistic Metric",
|
| 2572 |
+
"column": "Arena Photorealistic Elo",
|
| 2573 |
+
},
|
| 2574 |
+
{
|
| 2575 |
+
"id": "arena_art",
|
| 2576 |
+
"name": "Arena ELO - Art Metric",
|
| 2577 |
+
"column": "Arena Art Elo",
|
| 2578 |
+
},
|
| 2579 |
+
{
|
| 2580 |
+
"id": "arena_portraits",
|
| 2581 |
+
"name": "Arena ELO - Portraits Metric",
|
| 2582 |
+
"column": "Arena Portraits Elo",
|
| 2583 |
+
},
|
| 2584 |
+
{
|
| 2585 |
+
"id": "arena_text",
|
| 2586 |
+
"name": "Arena ELO - Text Rendering Metric",
|
| 2587 |
+
"column": "Arena Text Rendering Elo",
|
| 2588 |
},
|
| 2589 |
]
|
| 2590 |
|
|
|
|
| 2612 |
"datapoint_elo_object",
|
| 2613 |
],
|
| 2614 |
)
|
| 2615 |
+
aa_metric_ids = _metric_ids_for(aa_df, ["aa_elo"])
|
| 2616 |
+
arena_metric_ids = _metric_ids_for(
|
| 2617 |
+
arena_df,
|
| 2618 |
+
[
|
| 2619 |
+
"arena_elo",
|
| 2620 |
+
"arena_branding",
|
| 2621 |
+
"arena_3d",
|
| 2622 |
+
"arena_cartoon",
|
| 2623 |
+
"arena_photo",
|
| 2624 |
+
"arena_art",
|
| 2625 |
+
"arena_portraits",
|
| 2626 |
+
"arena_text",
|
| 2627 |
+
],
|
| 2628 |
+
)
|
| 2629 |
|
| 2630 |
datasets = [
|
| 2631 |
{
|
|
|
|
| 2654 |
},
|
| 2655 |
{
|
| 2656 |
"id": "artificial_analysis",
|
| 2657 |
+
"name": "Artificial Analysis",
|
| 2658 |
+
"data": aa_df,
|
| 2659 |
"columns": aa_display_columns,
|
| 2660 |
"metric_ids": aa_metric_ids,
|
| 2661 |
+
"note": "",
|
| 2662 |
+
"samples": None,
|
| 2663 |
+
},
|
| 2664 |
+
{
|
| 2665 |
+
"id": "arena_ai",
|
| 2666 |
+
"name": "Arena AI",
|
| 2667 |
+
"data": arena_df,
|
| 2668 |
+
"columns": arena_display_columns,
|
| 2669 |
+
"metric_ids": arena_metric_ids,
|
| 2670 |
+
"note": "",
|
| 2671 |
+
"samples": None,
|
| 2672 |
},
|
| 2673 |
]
|
| 2674 |
datasets = [dataset for dataset in datasets if dataset["metric_ids"]]
|
|
|
|
| 2677 |
(dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
|
| 2678 |
datasets[0]["id"] if datasets else None,
|
| 2679 |
)
|
| 2680 |
+
DEFAULT_METRIC_ID = None
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2681 |
|
| 2682 |
|
| 2683 |
custom_head = """
|
data/arena_ai_text_to_image_leaderboard.csv
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model,elo,product_branding_commercial_design_elo,3d_imaging_and_modeling_elo,cartoon_anime_and_fantasy_elo,photorealistic_and_cinematic_imagery_elo,art_elo,portraits_elo,text_rendering_elo
|
| 2 |
+
gpt-image-2 (medium),1381,1391,1361,1397,1380,1370,1429,1425
|
| 3 |
+
mai-image-2.6-preview,1336,1344,1362,1355,1325,1345,1365,1374
|
| 4 |
+
grok-imagine-image-2.0 (low),1316,1309,1268,1321,1327,1306,1371,1353
|
| 5 |
+
reve-2.1,1302,1314,1274,1310,1302,1289,1317,1341
|
| 6 |
+
muse-image,1282,1269,1255,1289,1293,1275,1313,1304
|
| 7 |
+
reve-2.0,1270,1279,1220,1275,1280,1259,1283,1297
|
| 8 |
+
gemini-3.1-flash-image (nano-banana-2) [web-search],1264,1271,1260,1268,1270,1251,1266,1295
|
| 9 |
+
seedream-5.0-pro,1258,1234,1246,1273,1280,1269,1291,1255
|
| 10 |
+
qwen-image-3.0-pro,1257,1258,1240,1271,1257,1257,1273,1289
|
| 11 |
+
mai-image-2.5,1256,1261,1248,1270,1252,1278,1259,1284
|
| 12 |
+
gemini-3.1-flash-lite-image (nano-banana-2-lite),1251,1250,1225,1258,1260,1257,1248,1275
|
| 13 |
+
gemini-3-pro-image-2k (nano-banana-pro),1246,1246,1231,1242,1260,1225,1258,1270
|
| 14 |
+
gpt-image-1.5-high-fidelity,1239,1241,1216,1243,1249,1224,1261,1254
|
| 15 |
+
gemini-3-pro-image-preview (nano-banana-pro),1232,1237,1209,1238,1243,1216,1236,1254
|
| 16 |
+
ideogram-4.0-quality,1204,1224,1180,1195,1203,1199,1201,1238
|
| 17 |
+
qwen-image-2.0-pro-2026-06-22,1191,1187,1198,1191,1205,1177,1216,1201
|
| 18 |
+
uni-1.1-max,1188,1197,1172,1194,1188,1195,1158,1217
|
| 19 |
+
mai-image-2,1183,1175,1179,1180,1203,1177,1207,1175
|
| 20 |
+
uni-1.1,1181,1181,1167,1189,1180,1188,1176,1202
|
| 21 |
+
Cosmos3-Super-Text2Image (Agentic),1175,1162,1170,1191,1189,1195,1212,1151
|
| 22 |
+
grok-imagine-image,1171,1175,1176,1177,1179,1179,1187,1185
|
| 23 |
+
recraft-v4.1-utility-pro,1169,1167,1147,1177,1180,1178,1169,1162
|
| 24 |
+
flux-2-max,1162,1165,1161,1169,1161,1170,1157,1167
|
| 25 |
+
grok-imagine-image-pro,1161,1168,1165,1163,1169,1165,1172,1182
|
| 26 |
+
flux-2-flex,1157,1173,1172,1162,1147,1156,1145,1172
|
| 27 |
+
flux-2-pro,1155,1157,1150,1161,1152,1163,1146,1157
|
| 28 |
+
Cosmos3-Super-Text2Image,1155,1129,1166,1177,1155,1173,1176,1122
|
| 29 |
+
reve-v1.5,1154,1182,1143,1141,1147,1144,1149,1185
|
| 30 |
+
hunyuan-image-3.0,1151,1153,1168,1163,1152,1153,1146,1150
|
| 31 |
+
gemini-2.5-flash-image-preview (nano-banana),1150,1152,1142,1145,1157,1145,1168,1154
|
| 32 |
+
imagen-ultra-4.0-generate-001,1148,1151,1141,1141,1156,1128,1168,1148
|
| 33 |
+
seedream-4.5,1147,1140,1147,1149,1166,1154,1166,1141
|
| 34 |
+
flux-2-dev,1145,1155,1149,1144,1143,1150,1133,1159
|
| 35 |
+
seedream-4-2k,1140,1134,1150,1149,1151,1164,1142,1143
|
| 36 |
+
seedream-5.0-lite,1137,1129,1138,1140,1154,1137,1150,1134
|
| 37 |
+
wan2.6-t2i,1136,1148,1146,1146,1128,1156,1123,1149
|
| 38 |
+
recraft-v4.1-pro,1130,1138,1111,1122,1143,1154,1144,1156
|
| 39 |
+
imagen-4.0-generate-001,1129,1129,1123,1120,1136,1110,1145,1123
|
| 40 |
+
qwen-image-2512,1125,1128,1135,1125,1135,1133,1143,1122
|
| 41 |
+
krea-2-medium,1122,1119,1132,1140,1127,1152,1148,1124
|
| 42 |
+
wan2.5-t2i-preview,1117,1138,1135,1120,1102,1120,1094,1142
|
| 43 |
+
hidream-o1-image,1117,1128,1121,1121,1111,1122,1106,1135
|
| 44 |
+
seedream-4-fal,1116,1128,1138,1109,1121,1119,1116,1124
|
| 45 |
+
gpt-image-1,1115,1115,1114,1115,1115,1115,1115,1115
|
| 46 |
+
recraft-v4,1114,1126,1076,1111,1119,1123,1119,1139
|
| 47 |
+
seedream-4-high-res-fal,1113,1113,1121,1109,1129,1112,1133,1110
|
| 48 |
+
krea-2-turbo,1111,1109,1133,1122,1123,1139,1138,1097
|
| 49 |
+
gpt-image-1-mini,1109,1109,1108,1108,1114,1113,1112,1104
|
| 50 |
+
krea-2-large,1107,1104,1113,1112,1121,1116,1117,1111
|
| 51 |
+
wan2.7-image-pro,1103,1114,1080,1080,1123,1079,1127,1118
|
| 52 |
+
wan2.7-image,1100,1110,1082,1075,1117,1075,1125,1114
|
| 53 |
+
mai-image-1,1093,1086,1096,1100,1103,1104,1095,1070
|
| 54 |
+
seedream-3,1082,1070,1094,1081,1102,1092,1102,1052
|
| 55 |
+
z-image-turbo,1082,1060,1066,1067,1115,1074,1140,1058
|
| 56 |
+
flux-1-kontext-max,1074,1082,1079,1079,1083,1081,1085,1071
|
| 57 |
+
flux-2-klein-9b,1070,1070,1078,1064,1090,1079,1092,1057
|
| 58 |
+
qwen-image-prompt-extend,1061,1087,1082,1062,1047,1055,1051,1082
|
| 59 |
+
flux-1-kontext-pro,1059,1068,1062,1058,1066,1067,1065,1053
|
| 60 |
+
imagen-3.0-generate-002,1058,1057,1046,1048,1075,1059,1077,1045
|
| 61 |
+
qwen-image,1057,1068,1064,1052,1062,1055,1076,1047
|
| 62 |
+
ideogram-v3-quality,1049,1082,1029,1027,1051,1039,1052,1070
|
| 63 |
+
photon,1035,1052,1033,1026,1035,1046,1023,1041
|
| 64 |
+
p-image,1034,1031,1031,1014,1074,1041,1100,1021
|
| 65 |
+
flux-2-klein-4b,1030,1026,1030,1023,1047,1028,1054,1015
|
| 66 |
+
runway-gen4,1025,1027,1013,1008,1041,1019,1022,1019
|
| 67 |
+
recraft-v3,1021,1027,1011,978,1054,991,1040,1005
|
| 68 |
+
flux-1.1-pro,1016,1012,1051,1016,1044,1044,1043,995
|
| 69 |
+
lucid-origin,1013,1020,1015,1007,1035,1031,1043,1005
|
| 70 |
+
ideogram-v2,1013,1054,990,995,1019,1017,1019,1046
|
| 71 |
+
glm-image,1010,1036,1018,1003,1000,1013,1000,1033
|
| 72 |
+
gemini-2.0-flash-preview-image-generation,975,1019,976,954,963,965,961,1007
|
| 73 |
+
flux-1-dev-fp8,969,991,978,957,990,978,999,963
|
| 74 |
+
dall-e-3,968,987,987,988,940,1010,924,969
|
| 75 |
+
flux-1-kontext-dev,940,966,943,924,935,927,944,953
|
| 76 |
+
stable-diffusion-v35-large,938,937,964,946,928,998,941,921
|
| 77 |
+
bagel,898,887,866,885,923,911,926,875
|
data/artificial_analysis_text_to_image_leaderboard.csv
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model,elo,price_per_image_usd
|
| 2 |
+
GPT Image 2 (high),1370,0.211
|
| 3 |
+
Reve 2.1,1324,0.2
|
| 4 |
+
Nano Banana 2 (Gemini 3.1 Flash Image Preview),1322,0.067
|
| 5 |
+
GPT Image 1.5 (high),1312,0.133
|
| 6 |
+
MAI-Image-2.5,1306,0.0481
|
| 7 |
+
Nano Banana Pro (Gemini 3 Pro Image),1298,0.134
|
| 8 |
+
Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image),1292,0.0336
|
| 9 |
+
Seedream 5.0 Pro,1281,0.09
|
| 10 |
+
grok-imagine-image-quality,1236,0.05
|
| 11 |
+
Qwen Image 2.0 Pro (2026-04-22),1235,0.075
|
| 12 |
+
FLUX.2 [max],1231,0.07
|
| 13 |
+
MAI-Image-2.5-Flash,1230,0.02
|
| 14 |
+
HiDream-O1-Image-1.5,1227,0.08
|
| 15 |
+
Seedream 4.0,1224,0.03
|
| 16 |
+
Luma UNI 1 Max,1223,0.1
|
| 17 |
+
FLUX.2 [flex],1222,0.06
|
| 18 |
+
Krea 2 Medium Turbo,1222,0.015
|
| 19 |
+
Krea 2 Large,1220,0.06
|
| 20 |
+
grok-imagine-image,1218,0.02
|
| 21 |
+
Recraft V4.1 Utility,1217,0.035
|
| 22 |
+
Ideogram 4.0,1217,0.06
|
| 23 |
+
Recraft V4.1 Utility Pro,1217,0.21
|
| 24 |
+
Ideogram 4.0 (Quality),1214,0.1
|
| 25 |
+
Wan2.6 Text to Image,1213,0.03
|
| 26 |
+
Krea 2 Medium,1212,0.03
|
| 27 |
+
MAI-Image-2,1210,0.035
|
| 28 |
+
FLUX.2 [pro],1208,0.03
|
| 29 |
+
Wan 2.6 Image,1206,0.03
|
| 30 |
+
Seedream 4.5,1205,0.04
|
| 31 |
+
GPT Image 1 (high),1205,0.167
|
| 32 |
+
Luma UNI 1,1205,0.0404
|
| 33 |
+
FLUX.2 [dev] Turbo,1199,0.008
|
| 34 |
+
FLUX.2 [dev],1199,0.012
|
| 35 |
+
Seedream 5.0 Lite,1199,0.035
|
| 36 |
+
Ideogram 4.0 Fast (Quality),1199,0.0175
|
| 37 |
+
P-Image-Ideogram (High),1198,0.015
|
| 38 |
+
Recraft V4 Pro,1195,0.25
|
| 39 |
+
Recraft V4.1,1191,0.035
|
| 40 |
+
Nano Banana (Gemini 2.5 Flash Image),1189,0.039
|
| 41 |
+
Recraft V4.1 Pro,1189,0.21
|
| 42 |
+
Imagen 4 Ultra,1189,0.06
|
| 43 |
+
Ideogram 4.0 Instant,1187,0.0075
|
| 44 |
+
Wan 2.7 Pro,1184,0.064
|
| 45 |
+
Recraft V4,1182,0.04
|
| 46 |
+
MAI-Image-2-Efficient,1182,0.022
|
| 47 |
+
P-Image-Ideogram (Medium),1181,0.01
|
| 48 |
+
FLUX.2 [dev] Flash,1179,0.005
|
| 49 |
+
HiDream-O1-Image,1176,
|
| 50 |
+
Ideogram 4.0 Fast,1176,0.0105
|
| 51 |
+
ImagineArt 2.0,1173,0.03
|
| 52 |
+
Qwen Image Max 2512,1173,0.02
|
| 53 |
+
Wan 2.7,1170,0.026
|
| 54 |
+
ImagineArt 1.5 Preview,1165,0.03
|
| 55 |
+
P-Image-Ideogram (Low),1165,0.0075
|
| 56 |
+
Seedream 3.0,1160,0.03
|
| 57 |
+
Vivago 2.1,1152,0.035
|
| 58 |
+
HunyuanImage 3.0 Instruct (Fal),1151,0.09
|
| 59 |
+
Wan 2.5 Preview,1150,0.021
|
| 60 |
+
FLUX.2 [klein] 9B,1146,0.015
|
| 61 |
+
FLUX.1 Kontext [max],1142,0.08
|
| 62 |
+
Kolors 2.1,1142,0.014
|
| 63 |
+
image-1,1141,0.04
|
| 64 |
+
HunyuanImage 3.0 (Fal),1140,0.1
|
| 65 |
+
Qwen Image 2.0 (2026-03-03),1135,0.035
|
| 66 |
+
Z-Image Turbo,1130,0.005
|
| 67 |
+
Vivago 2.0,1125,
|
| 68 |
+
Imagen 3 (v002),1124,0.04
|
| 69 |
+
Lucid Origin Ultra,1122,0.0867
|
| 70 |
+
Eigen Image,1122,0.025
|
| 71 |
+
Imagen 4 Standard,1120,0.04
|
| 72 |
+
Reve Image (Halfmoon),1116,
|
| 73 |
+
Kling Image 3.0 Omni,1115,0.028
|
| 74 |
+
Vidu Q2,1112,0.03
|
| 75 |
+
P-Image-Ideogram (Very Low),1111,0.003
|
| 76 |
+
Qwen Image Plus 2601,1110,0.03
|
| 77 |
+
GPT Image 1 Mini (medium),1110,0.011
|
| 78 |
+
Lucid Origin Fast,1110,0.0179
|
| 79 |
+
FLUX.1 Kontext [pro],1109,0.04
|
| 80 |
+
Dreamina 3.1,1108,0.03
|
| 81 |
+
FLUX1.1 [pro] Ultra,1104,0.06
|
| 82 |
+
Ideogram 3.0,1101,0.06
|
| 83 |
+
FLUX.2 [klein] Base 9B,1099,0.011
|
| 84 |
+
Imagen 4 Fast,1098,0.02
|
| 85 |
+
P-Image,1093,0.005
|
| 86 |
+
FLUX1.1 [pro],1092,0.04
|
| 87 |
+
Midjourney v7 Alpha,1092,
|
| 88 |
+
Ideogram v2,1084,0.08
|
| 89 |
+
FLUX.1 [pro],1083,0.05
|
| 90 |
+
Qwen Image,1082,0.02
|
| 91 |
+
SRPO,1080,0.026
|
| 92 |
+
HunyuanImage 2.1,1078,0.1
|
| 93 |
+
Midjourney v6,1076,
|
| 94 |
+
Ideogram v2 Turbo,1075,0.05
|
| 95 |
+
FIBO,1075,0.04
|
| 96 |
+
Recraft V3,1075,0.04
|
| 97 |
+
Luma Photon,1074,0.019
|
| 98 |
+
HiDream-O1-Image-Dev,1071,0.005
|
| 99 |
+
Image-01,1070,0.01
|
| 100 |
+
HiDream-I1-Dev,1070,0.024
|
| 101 |
+
GLM-Image,1068,0.05
|
| 102 |
+
Z-Image Base,1064,0.01
|
| 103 |
+
MAI Image 1,1064,
|
| 104 |
+
HiDream-I1-Fast,1063,0.012
|
| 105 |
+
FLUX.2 [klein] 4B,1057,0.014
|
| 106 |
+
Midjourney v6.1,1057,
|
| 107 |
+
Infinity 8B,1056,0.0017
|
| 108 |
+
LongCat Image,1053,0.13
|
| 109 |
+
Stable Diffusion 3 Large,1042,
|
| 110 |
+
FLUX.1 [dev],1041,0.025
|
| 111 |
+
Phoenix 1.0 Ultra,1041,0.0762
|
| 112 |
+
FLUX.1 Krea [dev],1036,0.025
|
| 113 |
+
Stable Diffusion 3.5 Large Turbo,1034,0.04
|
| 114 |
+
Stable Diffusion 3.5 Large,1034,0.065
|
| 115 |
+
Phoenix 0.9 Ultra,1028,0.0762
|
| 116 |
+
Ideogram v2a Turbo,1028,0.025
|
| 117 |
+
Ideogram v2a,1027,0.04
|
| 118 |
+
Playground v3 (beta),1015,0.0105
|
| 119 |
+
Krea 1,1013,
|
| 120 |
+
Phoenix 1.0 Fast,1007,0.0194
|
| 121 |
+
Luma Photon Flash,1004,0.005
|
| 122 |
+
FLUX.1 [schnell],1000,0.003
|
| 123 |
+
Step Image Edit 2,1000,0.003
|
| 124 |
+
Runway Gen-4 Image,992,0.08
|
| 125 |
+
Recraft 20B,984,0.022
|
| 126 |
+
Lumina Image v2,978,0.075
|
| 127 |
+
Playground v2.5,972,0.08
|
| 128 |
+
FLUX.2 [klein] Base 4B,972,0.009
|
| 129 |
+
DALLE 3,968,0.04
|
| 130 |
+
DALLE 3 HD,967,0.08
|
| 131 |
+
Stable Diffusion 3.5 Medium,963,0.02
|
| 132 |
+
Sana Sprint 1.6B,939,0.0015
|
| 133 |
+
Amazon Titan G1 v2 (Standard),923,0.01
|
| 134 |
+
Stable Diffusion 3 Medium,922,0.035
|
| 135 |
+
Stable Diffusion 3 Large Turbo,917,
|
| 136 |
+
Amazon Titan G1 (Standard),914,0.01
|
| 137 |
+
Stable Diffusion 1.6,914,0.009
|
| 138 |
+
OmniGen V2,911,0.15
|
| 139 |
+
SDXL Lightning,909,0.0019
|
| 140 |
+
Bagel,908,0.1
|
| 141 |
+
Bria 3.2,906,0.04
|
| 142 |
+
Stable Diffusion XL 1.0,884,0.009
|
| 143 |
+
Stable Diffusion 2.1,754,0.0039
|
| 144 |
+
DALLE 2,745,0.02
|
| 145 |
+
Janus Pro,720,0.0
|
| 146 |
+
Stable Diffusion 1.5,665,0.044
|
model_display.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""UI-only model display names.
|
| 2 |
+
|
| 3 |
+
Raw CSV / sample IDs stay unchanged; this maps aliases to a shared label
|
| 4 |
+
shown in leaderboards, Pareto plots, dropdowns, and sample headers.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import re
|
| 10 |
+
|
| 11 |
+
# Explicit aliases (any leaderboard / sample id) -> shared display label.
|
| 12 |
+
MODEL_DISPLAY_NAMES = {
|
| 13 |
+
# FLUX
|
| 14 |
+
"flux_2_pro": "FLUX.2 [pro]",
|
| 15 |
+
"FLUX.2 [pro]": "FLUX.2 [pro]",
|
| 16 |
+
"flux_2_max": "FLUX.2 [max]",
|
| 17 |
+
"FLUX.2 [max]": "FLUX.2 [max]",
|
| 18 |
+
"flux_2_flex": "FLUX.2 [flex]",
|
| 19 |
+
"FLUX.2 [flex]": "FLUX.2 [flex]",
|
| 20 |
+
"flux_2_dev": "FLUX.2 [dev]",
|
| 21 |
+
"FLUX.2 [dev]": "FLUX.2 [dev]",
|
| 22 |
+
"FLUX.2 [dev] Turbo": "FLUX.2 [dev] Turbo",
|
| 23 |
+
"FLUX.2 [dev] Flash": "FLUX.2 [dev] Flash",
|
| 24 |
+
"flux_1_1_pro": "FLUX1.1 [pro]",
|
| 25 |
+
"FLUX1.1 [pro]": "FLUX1.1 [pro]",
|
| 26 |
+
"flux_1_1_pro_ultra": "FLUX1.1 [pro] Ultra",
|
| 27 |
+
"FLUX1.1 [pro] Ultra": "FLUX1.1 [pro] Ultra",
|
| 28 |
+
"flux_dev": "FLUX.1 [dev]",
|
| 29 |
+
"FLUX.1 [dev]": "FLUX.1 [dev]",
|
| 30 |
+
"flux_schnell": "FLUX.1 [schnell]",
|
| 31 |
+
"FLUX.1 [schnell]": "FLUX.1 [schnell]",
|
| 32 |
+
"flux_krea": "FLUX.1 Krea [dev]",
|
| 33 |
+
"FLUX.1 Krea [dev]": "FLUX.1 Krea [dev]",
|
| 34 |
+
"FLUX.1 [pro]": "FLUX.1 [pro]",
|
| 35 |
+
"FLUX.1 Kontext [pro]": "FLUX.1 Kontext [pro]",
|
| 36 |
+
"FLUX.1 Kontext [max]": "FLUX.1 Kontext [max]",
|
| 37 |
+
# GPT Image
|
| 38 |
+
"gpt_image_2": "GPT Image 2",
|
| 39 |
+
"GPT Image 2 (high)": "GPT Image 2",
|
| 40 |
+
"gpt_image_1_5": "GPT Image 1.5",
|
| 41 |
+
"GPT Image 1.5 (high)": "GPT Image 1.5",
|
| 42 |
+
"gpt_image_1": "GPT Image 1",
|
| 43 |
+
"GPT Image 1 (high)": "GPT Image 1",
|
| 44 |
+
"GPT Image 1 Mini (medium)": "GPT Image 1 Mini",
|
| 45 |
+
# Nano Banana / Gemini
|
| 46 |
+
"nano_banana_2_0": "Nano Banana 2",
|
| 47 |
+
"nano_banana_2": "Nano Banana 2",
|
| 48 |
+
"Nano Banana 2 (Gemini 3.1 Flash Image Preview)": "Nano Banana 2",
|
| 49 |
+
"Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)": "Nano Banana 2 Lite",
|
| 50 |
+
"nano_banana_pro": "Nano Banana Pro",
|
| 51 |
+
"Nano Banana Pro (Gemini 3 Pro Image)": "Nano Banana Pro",
|
| 52 |
+
"Nano Banana (Gemini 2.5 Flash Image)": "Nano Banana",
|
| 53 |
+
# Seedream
|
| 54 |
+
"seedream_5_0": "Seedream 5.0",
|
| 55 |
+
"Seedream 5.0 Pro": "Seedream 5.0 Pro",
|
| 56 |
+
"Seedream 5.0 Lite": "Seedream 5.0 Lite",
|
| 57 |
+
"seedream_4_5": "Seedream 4.5",
|
| 58 |
+
"Seedream 4.5": "Seedream 4.5",
|
| 59 |
+
"seedream_4_0": "Seedream 4.0",
|
| 60 |
+
"Seedream 4.0": "Seedream 4.0",
|
| 61 |
+
"seedream_3": "Seedream 3.0",
|
| 62 |
+
"Seedream 3.0": "Seedream 3.0",
|
| 63 |
+
# Qwen
|
| 64 |
+
"qwen_image": "Qwen Image",
|
| 65 |
+
"Qwen Image": "Qwen Image",
|
| 66 |
+
"qwen_image_2_0_pro": "Qwen Image 2.0 Pro",
|
| 67 |
+
"Qwen Image 2.0 Pro (2026-04-22)": "Qwen Image 2.0 Pro",
|
| 68 |
+
"Qwen Image 2.0 (2026-03-03)": "Qwen Image 2.0",
|
| 69 |
+
"qwen_image_2512": "Qwen Image 2512",
|
| 70 |
+
"Qwen Image Max 2512": "Qwen Image 2512",
|
| 71 |
+
"qwen_image_fast": "Qwen Image Fast",
|
| 72 |
+
"Qwen Image Plus 2601": "Qwen Image Plus 2601",
|
| 73 |
+
# Ideogram
|
| 74 |
+
"ideogram_4_0_quality": "Ideogram 4.0 Quality",
|
| 75 |
+
"Ideogram 4.0 (Quality)": "Ideogram 4.0 Quality",
|
| 76 |
+
"Ideogram 4.0": "Ideogram 4.0",
|
| 77 |
+
"Ideogram 4.0 Fast": "Ideogram 4.0 Fast",
|
| 78 |
+
"Ideogram 4.0 Fast (Quality)": "Ideogram 4.0 Fast Quality",
|
| 79 |
+
"Ideogram 4.0 Instant": "Ideogram 4.0 Instant",
|
| 80 |
+
"Ideogram 3.0": "Ideogram 3.0",
|
| 81 |
+
# Imagen
|
| 82 |
+
"imagen_4_0": "Imagen 4",
|
| 83 |
+
"imagen_4": "Imagen 4",
|
| 84 |
+
"Imagen 4 Standard": "Imagen 4",
|
| 85 |
+
"imagen_4_0_ultra": "Imagen 4 Ultra",
|
| 86 |
+
"imagen_4_ultra": "Imagen 4 Ultra",
|
| 87 |
+
"Imagen 4 Ultra": "Imagen 4 Ultra",
|
| 88 |
+
"imagen_4_fast": "Imagen 4 Fast",
|
| 89 |
+
"Imagen 4 Fast": "Imagen 4 Fast",
|
| 90 |
+
"Imagen 3 (v002)": "Imagen 3",
|
| 91 |
+
# HiDream
|
| 92 |
+
"hidream_i1_dev": "HiDream-I1 Dev",
|
| 93 |
+
"HiDream-I1-Dev": "HiDream-I1 Dev",
|
| 94 |
+
"hidream_i1_fast": "HiDream-I1 Fast",
|
| 95 |
+
"HiDream-I1-Fast": "HiDream-I1 Fast",
|
| 96 |
+
"hidream_i1_full": "HiDream-I1 Full",
|
| 97 |
+
"HiDream-O1-Image": "HiDream-O1",
|
| 98 |
+
"HiDream-O1-Image-1.5": "HiDream-O1 1.5",
|
| 99 |
+
"HiDream-O1-Image-Dev": "HiDream-O1 Dev",
|
| 100 |
+
# Reve
|
| 101 |
+
"reve_2_1": "Reve 2.1",
|
| 102 |
+
"Reve 2.1": "Reve 2.1",
|
| 103 |
+
"Reve Image (Halfmoon)": "Reve Image",
|
| 104 |
+
# P-Image
|
| 105 |
+
"p_image": "P-Image",
|
| 106 |
+
"P-Image": "P-Image",
|
| 107 |
+
"p_image_2_ideogram_very_low_1k": "P-Image-Ideogram Very Low 1K",
|
| 108 |
+
"p_image_2_ideogram_very_low_2k": "P-Image-Ideogram Very Low 2K",
|
| 109 |
+
"P-Image-Ideogram (Very Low)": "P-Image-Ideogram Very Low",
|
| 110 |
+
"p_image_2_ideogram_low_1k": "P-Image-Ideogram Low 1K",
|
| 111 |
+
"p_image_2_ideogram_low_2k": "P-Image-Ideogram Low 2K",
|
| 112 |
+
"P-Image-Ideogram (Low)": "P-Image-Ideogram Low",
|
| 113 |
+
"p_image_2_ideogram_medium_1k": "P-Image-Ideogram Medium 1K",
|
| 114 |
+
"p_image_2_ideogram_medium_2k": "P-Image-Ideogram Medium 2K",
|
| 115 |
+
"P-Image-Ideogram (Medium)": "P-Image-Ideogram Medium",
|
| 116 |
+
"p_image_2_ideogram_high_1k": "P-Image-Ideogram High 1K",
|
| 117 |
+
"p_image_2_ideogram_high_2k": "P-Image-Ideogram High 2K",
|
| 118 |
+
"P-Image-Ideogram (High)": "P-Image-Ideogram High",
|
| 119 |
+
# Others overlapping P-Bench
|
| 120 |
+
"z_image": "Z-Image",
|
| 121 |
+
"Z-Image Turbo": "Z-Image Turbo",
|
| 122 |
+
"Z-Image Base": "Z-Image Base",
|
| 123 |
+
"glm_image": "GLM-Image",
|
| 124 |
+
"GLM-Image": "GLM-Image",
|
| 125 |
+
"hunyuanimage_3_0": "HunyuanImage 3.0",
|
| 126 |
+
"hunyuan_image_3": "HunyuanImage 3.0",
|
| 127 |
+
"HunyuanImage 3.0 (Fal)": "HunyuanImage 3.0",
|
| 128 |
+
"HunyuanImage 3.0 Instruct (Fal)": "HunyuanImage 3.0 Instruct",
|
| 129 |
+
"HunyuanImage 2.1": "HunyuanImage 2.1",
|
| 130 |
+
"wan_2_2_image": "Wan 2.2 Image",
|
| 131 |
+
"Wan 2.6 Image": "Wan 2.6 Image",
|
| 132 |
+
"Wan2.6 Text to Image": "Wan 2.6 Text to Image",
|
| 133 |
+
"kling_v2_1": "Kling v2.1",
|
| 134 |
+
"Kling Image 3.0 Omni": "Kling Image 3.0 Omni",
|
| 135 |
+
"juggernaut_base_flux": "Juggernaut Base FLUX",
|
| 136 |
+
"juggernaut_pro_flux": "Juggernaut Pro FLUX",
|
| 137 |
+
"juggernaut_lightning_flux": "Juggernaut Lightning FLUX",
|
| 138 |
+
"bria_4_fibo": "Bria FIBO",
|
| 139 |
+
"Bria 3.2": "Bria 3.2",
|
| 140 |
+
"sdxl": "SDXL 1.0",
|
| 141 |
+
"Stable Diffusion XL 1.0": "SDXL 1.0",
|
| 142 |
+
"sdxl_lightning": "SDXL Lightning",
|
| 143 |
+
"SDXL Lightning": "SDXL Lightning",
|
| 144 |
+
# Arena AI (kebab / arena.ai ids)
|
| 145 |
+
"gpt-image-2 (medium)": "GPT Image 2",
|
| 146 |
+
"gpt-image-1.5-high-fidelity": "GPT Image 1.5",
|
| 147 |
+
"gpt-image-1": "GPT Image 1",
|
| 148 |
+
"gpt-image-1-mini": "GPT Image 1 Mini",
|
| 149 |
+
"mai-image-2.6-preview": "MAI-Image-2.6 Preview",
|
| 150 |
+
"mai-image-2.5": "MAI-Image-2.5",
|
| 151 |
+
"mai-image-2": "MAI-Image-2",
|
| 152 |
+
"mai-image-1": "MAI Image 1",
|
| 153 |
+
"grok-imagine-image-2.0 (low)": "Grok Imagine Image 2.0",
|
| 154 |
+
"grok-imagine-image": "Grok Imagine Image",
|
| 155 |
+
"grok-imagine-image-pro": "Grok Imagine Image Pro",
|
| 156 |
+
"reve-2.1": "Reve 2.1",
|
| 157 |
+
"reve-2.0": "Reve 2.0",
|
| 158 |
+
"reve-v1.5": "Reve 1.5",
|
| 159 |
+
"muse-image": "Muse Image",
|
| 160 |
+
"gemini-3.1-flash-image (nano-banana-2) [web-search]": "Nano Banana 2",
|
| 161 |
+
"gemini-3.1-flash-lite-image (nano-banana-2-lite)": "Nano Banana 2 Lite",
|
| 162 |
+
"gemini-3-pro-image-2k (nano-banana-pro)": "Nano Banana Pro 2K",
|
| 163 |
+
"gemini-3-pro-image-preview (nano-banana-pro)": "Nano Banana Pro",
|
| 164 |
+
"gemini-2.5-flash-image-preview (nano-banana)": "Nano Banana",
|
| 165 |
+
"gemini-2.0-flash-preview-image-generation": "Gemini 2.0 Flash Image",
|
| 166 |
+
"seedream-5.0-pro": "Seedream 5.0 Pro",
|
| 167 |
+
"seedream-5.0-lite": "Seedream 5.0 Lite",
|
| 168 |
+
"seedream-4.5": "Seedream 4.5",
|
| 169 |
+
"seedream-4-2k": "Seedream 4.0 2K",
|
| 170 |
+
"seedream-4-fal": "Seedream 4.0",
|
| 171 |
+
"seedream-4-high-res-fal": "Seedream 4.0 High Res",
|
| 172 |
+
"seedream-3": "Seedream 3.0",
|
| 173 |
+
"qwen-image-3.0-pro": "Qwen Image 3.0 Pro",
|
| 174 |
+
"qwen-image-2.0-pro-2026-06-22": "Qwen Image 2.0 Pro",
|
| 175 |
+
"qwen-image-2512": "Qwen Image 2512",
|
| 176 |
+
"qwen-image-prompt-extend": "Qwen Image Prompt Extend",
|
| 177 |
+
"qwen-image": "Qwen Image",
|
| 178 |
+
"uni-1.1-max": "Luma UNI 1 Max",
|
| 179 |
+
"uni-1.1": "Luma UNI 1",
|
| 180 |
+
"Cosmos3-Super-Text2Image (Agentic)": "Cosmos3 Super Text2Image (Agentic)",
|
| 181 |
+
"Cosmos3-Super-Text2Image": "Cosmos3 Super Text2Image",
|
| 182 |
+
"recraft-v4.1-utility-pro": "Recraft V4.1 Utility Pro",
|
| 183 |
+
"recraft-v4.1-pro": "Recraft V4.1 Pro",
|
| 184 |
+
"recraft-v4": "Recraft V4",
|
| 185 |
+
"recraft-v3": "Recraft V3",
|
| 186 |
+
"flux-2-max": "FLUX.2 [max]",
|
| 187 |
+
"flux-2-flex": "FLUX.2 [flex]",
|
| 188 |
+
"flux-2-pro": "FLUX.2 [pro]",
|
| 189 |
+
"flux-2-dev": "FLUX.2 [dev]",
|
| 190 |
+
"flux-2-klein-9b": "FLUX.2 [klein] 9B",
|
| 191 |
+
"flux-2-klein-4b": "FLUX.2 [klein] 4B",
|
| 192 |
+
"flux-1-kontext-max": "FLUX.1 Kontext [max]",
|
| 193 |
+
"flux-1-kontext-pro": "FLUX.1 Kontext [pro]",
|
| 194 |
+
"flux-1-kontext-dev": "FLUX.1 Kontext [dev]",
|
| 195 |
+
"flux-1.1-pro": "FLUX1.1 [pro]",
|
| 196 |
+
"flux-1-dev-fp8": "FLUX.1 [dev]",
|
| 197 |
+
"hunyuan-image-3.0": "HunyuanImage 3.0",
|
| 198 |
+
"imagen-ultra-4.0-generate-001": "Imagen 4 Ultra",
|
| 199 |
+
"imagen-4.0-generate-001": "Imagen 4",
|
| 200 |
+
"imagen-3.0-generate-002": "Imagen 3",
|
| 201 |
+
"wan2.6-t2i": "Wan 2.6 Text to Image",
|
| 202 |
+
"wan2.5-t2i-preview": "Wan 2.5 Preview",
|
| 203 |
+
"wan2.7-image-pro": "Wan 2.7 Pro",
|
| 204 |
+
"wan2.7-image": "Wan 2.7",
|
| 205 |
+
"ideogram-4.0-quality": "Ideogram 4.0 Quality",
|
| 206 |
+
"ideogram-v3-quality": "Ideogram 3.0",
|
| 207 |
+
"ideogram-v2": "Ideogram v2",
|
| 208 |
+
"krea-2-medium": "Krea 2 Medium",
|
| 209 |
+
"krea-2-turbo": "Krea 2 Medium Turbo",
|
| 210 |
+
"krea-2-large": "Krea 2 Large",
|
| 211 |
+
"hidream-o1-image": "HiDream-O1",
|
| 212 |
+
"z-image-turbo": "Z-Image Turbo",
|
| 213 |
+
"photon": "Luma Photon",
|
| 214 |
+
"p-image": "P-Image",
|
| 215 |
+
"runway-gen4": "Runway Gen-4 Image",
|
| 216 |
+
"lucid-origin": "Lucid Origin",
|
| 217 |
+
"glm-image": "GLM-Image",
|
| 218 |
+
"dall-e-3": "DALLE 3",
|
| 219 |
+
"stable-diffusion-v35-large": "Stable Diffusion 3.5 Large",
|
| 220 |
+
"bagel": "Bagel",
|
| 221 |
+
}
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _prettify_snake_case(model_id: str) -> str:
|
| 225 |
+
parts = [part for part in str(model_id).split("_") if part]
|
| 226 |
+
pretty = []
|
| 227 |
+
for part in parts:
|
| 228 |
+
if part.isdigit():
|
| 229 |
+
pretty.append(part)
|
| 230 |
+
elif re.fullmatch(r"\d+k", part, flags=re.IGNORECASE):
|
| 231 |
+
pretty.append(part.upper())
|
| 232 |
+
elif part.lower() in {"flux", "gpt", "sdxl", "glm"}:
|
| 233 |
+
pretty.append(part.upper())
|
| 234 |
+
else:
|
| 235 |
+
pretty.append(part.capitalize())
|
| 236 |
+
return " ".join(pretty)
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
def display_model_name(model_id) -> str:
|
| 240 |
+
"""Return the shared UI label for a raw model id / source name."""
|
| 241 |
+
if model_id is None:
|
| 242 |
+
return ""
|
| 243 |
+
raw = str(model_id).strip()
|
| 244 |
+
if not raw:
|
| 245 |
+
return ""
|
| 246 |
+
if raw in MODEL_DISPLAY_NAMES:
|
| 247 |
+
return MODEL_DISPLAY_NAMES[raw]
|
| 248 |
+
# Already a human label (spaces / punctuation) — keep as-is.
|
| 249 |
+
if re.search(r"[\s.\[\]()]", raw):
|
| 250 |
+
return raw
|
| 251 |
+
return _prettify_snake_case(raw)
|
ui.py
CHANGED
|
@@ -7,6 +7,8 @@ import gradio as gr
|
|
| 7 |
import pandas as pd
|
| 8 |
import plotly.graph_objects as go
|
| 9 |
|
|
|
|
|
|
|
| 10 |
_MASCOT_PATH = Path(__file__).resolve().parent / "pruna-mascot.png"
|
| 11 |
_MASCOT_DATA_URI = (
|
| 12 |
"data:image/png;base64,"
|
|
@@ -19,6 +21,7 @@ _MASCOT_DATA_URI = (
|
|
| 19 |
MAX_COMPARE_MODELS = 4
|
| 20 |
DEFAULT_COMPARE_PROMPTS = 3
|
| 21 |
MAX_COMPARE_PROMPTS = 8
|
|
|
|
| 22 |
|
| 23 |
ABOUT_OVERVIEW_CONTENT = """
|
| 24 |
# About P-Bench
|
|
@@ -43,8 +46,7 @@ across P-Bench.
|
|
| 43 |
2. It generates one image per prompt when the run succeeds. Not every model
|
| 44 |
has every prompt or every metric.
|
| 45 |
3. Quality is scored automatically (OneIG alignment, P-Judger) and, where
|
| 46 |
-
available, by human preference (Datapoint Elo, Rapidata Elo)
|
| 47 |
-
Artificial Analysis Elo.
|
| 48 |
4. Price per image and generation time are joined from the evaluation table.
|
| 49 |
|
| 50 |
## Current datasets
|
|
@@ -59,10 +61,6 @@ Prompt-image **alignment** on anime / stylization, human / portrait, and
|
|
| 59 |
general object prompts (100 prompts each). This is the alignment slice of
|
| 60 |
OneIG, not the full suite. Alignment Overall is the mean of the category
|
| 61 |
scores that exist for that row.
|
| 62 |
-
|
| 63 |
-
### Artificial Analysis Dataset
|
| 64 |
-
Artificial Analysis Elo from the evaluation table. Rapidata is a **metric**,
|
| 65 |
-
not a dataset.
|
| 66 |
"""
|
| 67 |
|
| 68 |
ABOUT_DETAILS_CONTENT = """
|
|
@@ -79,8 +77,6 @@ ABOUT_DETAILS_CONTENT = """
|
|
| 79 |
- **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.
|
| 80 |
Rapidata rejects prompts over 400 characters, so this Elo is on a subset
|
| 81 |
of each suite (see Setup). Rapidata is not a dataset.
|
| 82 |
-
- **Artificial Analysis Elo**: preference Elo from Artificial Analysis, shown
|
| 83 |
-
as its own dataset.
|
| 84 |
- **Generation time**: median and minimum generation time in seconds, as
|
| 85 |
reported in the evaluation table. This is not a p95, and we do not state
|
| 86 |
warm vs cold or concurrent load.
|
|
@@ -186,8 +182,23 @@ def _item(items, item_id):
|
|
| 186 |
return items[0] if items else None
|
| 187 |
|
| 188 |
|
| 189 |
-
def _dataset_choices(datasets):
|
| 190 |
-
return [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
|
| 192 |
|
| 193 |
def _metric_choices(datasets, metrics, dataset_id):
|
|
@@ -205,11 +216,21 @@ def _metric_choices(datasets, metrics, dataset_id):
|
|
| 205 |
|
| 206 |
|
| 207 |
def _coerce_metric(datasets, metrics, dataset_id, metric_id):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 208 |
choices = _metric_choices(datasets, metrics, dataset_id)
|
| 209 |
ids = [choice[1] for choice in choices]
|
| 210 |
if metric_id in ids:
|
| 211 |
return metric_id
|
| 212 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 213 |
|
| 214 |
|
| 215 |
def _model_choices(datasets, dataset_id):
|
|
@@ -217,42 +238,92 @@ def _model_choices(datasets, dataset_id):
|
|
| 217 |
data = dataset.get("data") if dataset else None
|
| 218 |
if data is None or "Model" not in getattr(data, "columns", []):
|
| 219 |
return []
|
| 220 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
|
| 222 |
|
| 223 |
def _view_title(datasets, metrics, dataset_id, metric_id):
|
| 224 |
dataset = _item(datasets, dataset_id)
|
| 225 |
-
metric = _item(metrics, metric_id)
|
| 226 |
dataset_name = dataset["name"] if dataset else "Dataset"
|
| 227 |
-
|
| 228 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 229 |
|
| 230 |
|
| 231 |
def _columns_for_metric(dataset, metric_column):
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 241 |
|
| 242 |
|
| 243 |
def resolve_view(datasets, metrics, dataset_id, metric_id):
|
| 244 |
dataset = _item(datasets, dataset_id)
|
| 245 |
-
|
| 246 |
-
metric = _item(metrics, metric_id)
|
| 247 |
-
if not dataset or not metric:
|
| 248 |
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 249 |
return {
|
| 250 |
"dataset": dataset,
|
| 251 |
"metric": metric,
|
| 252 |
-
"
|
|
|
|
| 253 |
"data": dataset["data"],
|
| 254 |
-
"columns":
|
| 255 |
-
"score_column":
|
|
|
|
| 256 |
"samples": dataset.get("samples"),
|
| 257 |
"note": dataset.get("note"),
|
| 258 |
}
|
|
@@ -272,6 +343,8 @@ def _format_leaderboard_cell(column, value):
|
|
| 272 |
except (TypeError, ValueError):
|
| 273 |
return escape(str(value))
|
| 274 |
if label in {"model", "platform", "endpoint owner", "optimized"}:
|
|
|
|
|
|
|
| 275 |
return escape(str(value))
|
| 276 |
try:
|
| 277 |
number = float(value)
|
|
@@ -384,6 +457,11 @@ def _filter_leaderboard(data, search_term, platform, owner, optimized, models=No
|
|
| 384 |
matches |= filtered[column].astype(str).str.contains(
|
| 385 |
search_term, case=False, na=False
|
| 386 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 387 |
filtered = filtered[matches]
|
| 388 |
|
| 389 |
for column, values in [
|
|
@@ -467,7 +545,14 @@ def _display_label(column):
|
|
| 467 |
"P-Judge Overall": "P-Judger (Pruna)",
|
| 468 |
"Datapoint Elo": "Datapoint Elo",
|
| 469 |
"Rapidata Elo": "Rapidata Elo",
|
| 470 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 471 |
"Raw Win Rate": "Raw win rate",
|
| 472 |
"Median Generation Time (s)": "Median generation time",
|
| 473 |
"Min Generation Time (s)": "Min generation time",
|
|
@@ -544,8 +629,12 @@ def _build_pareto_figure(
|
|
| 544 |
scores = scatter[score_column].astype(float).tolist()
|
| 545 |
on_frontier = _pareto_frontier_mask(x_values, scores)
|
| 546 |
|
| 547 |
-
dominated = scatter.loc[[not flag for flag in on_frontier]]
|
| 548 |
-
frontier = scatter.loc[on_frontier].sort_values(x_column)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 549 |
hover = (
|
| 550 |
"<b>%{text}</b><br>"
|
| 551 |
f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"
|
|
@@ -661,13 +750,37 @@ def _pareto_pair(data, score_column):
|
|
| 661 |
)
|
| 662 |
|
| 663 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 664 |
def _samples_html(samples, selected_models, num_prompts, seed=0):
|
| 665 |
if not samples:
|
| 666 |
-
return
|
| 667 |
-
'<div class="compare-empty">'
|
| 668 |
-
"Sample comparison is not available for this dataset yet."
|
| 669 |
-
"</div>"
|
| 670 |
-
)
|
| 671 |
models = list(selected_models or [])
|
| 672 |
available = samples.get("models") or []
|
| 673 |
models = [model for model in models if model in samples.get("images", {})]
|
|
@@ -722,9 +835,9 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
|
|
| 722 |
cells.append(
|
| 723 |
f"""
|
| 724 |
<div class="compare-cell">
|
| 725 |
-
<div class="compare-model-label">{escape(model)}</div>
|
| 726 |
<a href="{image_url}" target="_blank" rel="noopener noreferrer">
|
| 727 |
-
<img src="{image_url}" alt="{escape(model)} sample" loading="lazy" />
|
| 728 |
</a>
|
| 729 |
</div>
|
| 730 |
"""
|
|
@@ -755,38 +868,55 @@ def _note_markdown(note):
|
|
| 755 |
return note or ""
|
| 756 |
|
| 757 |
|
| 758 |
-
def _filter_row(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 759 |
metric_choices = _metric_choices(datasets, metrics, default_dataset_id)
|
| 760 |
model_choices = _model_choices(datasets, default_dataset_id)
|
|
|
|
|
|
|
|
|
|
| 761 |
with gr.Row(elem_classes="view-filters"):
|
| 762 |
dataset_dd = gr.Dropdown(
|
| 763 |
-
choices=_dataset_choices(datasets),
|
| 764 |
value=default_dataset_id,
|
| 765 |
label="Dataset",
|
| 766 |
type="value",
|
| 767 |
scale=2,
|
| 768 |
min_width=160,
|
| 769 |
)
|
| 770 |
-
metric_dd =
|
| 771 |
-
|
| 772 |
-
|
| 773 |
-
|
| 774 |
-
|
| 775 |
-
|
| 776 |
-
|
| 777 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 778 |
models_dd = gr.Dropdown(
|
| 779 |
choices=model_choices,
|
| 780 |
value=[],
|
| 781 |
multiselect=True,
|
| 782 |
label="Models",
|
| 783 |
type="value",
|
|
|
|
| 784 |
scale=3,
|
| 785 |
min_width=200,
|
| 786 |
)
|
| 787 |
title = gr.Markdown(
|
| 788 |
_title_markdown(
|
| 789 |
-
_view_title(datasets, metrics, default_dataset_id,
|
| 790 |
),
|
| 791 |
elem_classes="view-title",
|
| 792 |
)
|
|
@@ -800,14 +930,17 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 800 |
initial = resolve_view(datasets, metrics, default_dataset_id, default_metric_id)
|
| 801 |
initial_data = initial["data"]
|
| 802 |
initial_columns = initial["columns"]
|
| 803 |
-
|
| 804 |
initial_samples = initial.get("samples")
|
| 805 |
-
|
|
|
|
|
|
|
|
|
|
| 806 |
|
| 807 |
with gr.Tabs(elem_classes="main-tabs"):
|
| 808 |
with gr.TabItem("Leaderboards"):
|
| 809 |
lb_dataset, lb_metric, lb_models, lb_title = _filter_row(
|
| 810 |
-
datasets, metrics, default_dataset_id,
|
| 811 |
)
|
| 812 |
lb_note = gr.Markdown(_note_markdown(initial.get("note")))
|
| 813 |
platform_choices = _filter_choices(initial_data, "Platform")
|
|
@@ -847,7 +980,10 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 847 |
)
|
| 848 |
ranking = gr.HTML(
|
| 849 |
_leaderboard_html(
|
| 850 |
-
initial_data,
|
|
|
|
|
|
|
|
|
|
| 851 |
),
|
| 852 |
padding=False,
|
| 853 |
elem_classes="ranking-table-host",
|
|
@@ -855,65 +991,102 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 855 |
|
| 856 |
with gr.TabItem("Pareto Plots"):
|
| 857 |
pp_dataset, pp_metric, pp_models, pp_title = _filter_row(
|
| 858 |
-
datasets, metrics, default_dataset_id,
|
| 859 |
)
|
| 860 |
gr.Markdown(
|
| 861 |
"<span class='pareto-help'>"
|
| 862 |
"Green = on the frontier (lower cost or time at the same or better score). "
|
| 863 |
-
"Lavender = below the frontier."
|
|
|
|
| 864 |
"</span><br/>"
|
| 865 |
"<strong class='pareto-help-emphasis'>Hover a point to see which model it is.</strong>"
|
| 866 |
)
|
| 867 |
-
|
| 868 |
-
|
| 869 |
-
|
| 870 |
-
|
| 871 |
-
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
| 875 |
-
|
| 876 |
-
|
| 877 |
-
|
| 878 |
-
|
| 879 |
-
|
| 880 |
-
|
|
|
|
| 881 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 882 |
|
| 883 |
with gr.TabItem("Samples"):
|
| 884 |
-
|
| 885 |
-
datasets,
|
| 886 |
)
|
| 887 |
-
|
| 888 |
-
|
| 889 |
-
|
| 890 |
-
|
| 891 |
-
|
| 892 |
-
|
| 893 |
-
|
| 894 |
-
</p>
|
| 895 |
-
"""
|
| 896 |
)
|
| 897 |
-
with gr.
|
| 898 |
-
|
| 899 |
-
|
| 900 |
-
|
| 901 |
-
|
| 902 |
-
|
| 903 |
-
|
| 904 |
-
|
| 905 |
-
|
| 906 |
-
|
| 907 |
-
min_width=180,
|
| 908 |
-
elem_classes="compare-prompt-count",
|
| 909 |
-
)
|
| 910 |
-
shuffle_button = gr.Button(
|
| 911 |
-
"Shuffle prompts",
|
| 912 |
-
variant="primary",
|
| 913 |
-
scale=0,
|
| 914 |
-
min_width=140,
|
| 915 |
-
elem_classes="compare-shuffle",
|
| 916 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 917 |
gallery = gr.HTML(
|
| 918 |
value=_samples_html(
|
| 919 |
initial_samples, [], DEFAULT_COMPARE_PROMPTS, seed=0
|
|
@@ -925,33 +1098,49 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 925 |
with gr.TabItem("About"):
|
| 926 |
render_about()
|
| 927 |
|
| 928 |
-
def _synced_filters(dataset_id, metric_id, models):
|
| 929 |
-
|
|
|
|
|
|
|
|
|
|
| 930 |
model_choices = _model_choices(datasets, dataset_id)
|
| 931 |
-
|
|
|
|
| 932 |
metric_choices = _metric_choices(datasets, metrics, dataset_id)
|
| 933 |
title = _title_markdown(
|
| 934 |
_view_title(datasets, metrics, dataset_id, metric_id)
|
| 935 |
)
|
| 936 |
dataset_update = gr.update(value=dataset_id)
|
| 937 |
-
metric_update = gr.update(
|
|
|
|
|
|
|
| 938 |
models_update = gr.update(choices=model_choices, value=models)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 939 |
return (
|
| 940 |
dataset_id,
|
| 941 |
metric_id,
|
| 942 |
models,
|
|
|
|
| 943 |
dataset_update,
|
| 944 |
dataset_update,
|
| 945 |
-
|
| 946 |
metric_update,
|
| 947 |
metric_update,
|
| 948 |
-
metric_update,
|
| 949 |
-
models_update,
|
| 950 |
models_update,
|
| 951 |
models_update,
|
|
|
|
| 952 |
title,
|
| 953 |
title,
|
| 954 |
-
|
| 955 |
)
|
| 956 |
|
| 957 |
def _leaderboard_extras(data, platform_value, owner_value, optimized_value):
|
|
@@ -1009,28 +1198,38 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1009 |
optimized_value or [],
|
| 1010 |
models=models,
|
| 1011 |
)
|
|
|
|
|
|
|
|
|
|
| 1012 |
ranking_html = _leaderboard_html(
|
| 1013 |
filtered,
|
| 1014 |
view["columns"],
|
| 1015 |
-
|
| 1016 |
-
|
| 1017 |
)
|
| 1018 |
-
pareto_data = _filter_leaderboard(
|
| 1019 |
-
|
| 1020 |
-
)
|
| 1021 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1022 |
samples_html = _samples_html(
|
| 1023 |
-
|
| 1024 |
-
|
| 1025 |
int(num_prompts or DEFAULT_COMPARE_PROMPTS),
|
| 1026 |
int(seed or 0),
|
| 1027 |
)
|
| 1028 |
return (
|
| 1029 |
_note_markdown(view.get("note")),
|
| 1030 |
ranking_html,
|
| 1031 |
-
|
| 1032 |
-
next_time,
|
| 1033 |
samples_html,
|
|
|
|
| 1034 |
)
|
| 1035 |
|
| 1036 |
def on_dataset(
|
|
@@ -1044,7 +1243,9 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1044 |
num_prompts,
|
| 1045 |
seed,
|
| 1046 |
):
|
| 1047 |
-
synced = _synced_filters(
|
|
|
|
|
|
|
| 1048 |
dataset_id, metric_id, models = synced[:3]
|
| 1049 |
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1050 |
extras = _leaderboard_extras(
|
|
@@ -1061,7 +1262,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1061 |
num_prompts,
|
| 1062 |
seed,
|
| 1063 |
)
|
| 1064 |
-
return (*synced[
|
| 1065 |
|
| 1066 |
def on_metric(
|
| 1067 |
dataset_id,
|
|
@@ -1075,10 +1276,21 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1075 |
seed,
|
| 1076 |
):
|
| 1077 |
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
|
|
|
| 1078 |
title = _title_markdown(
|
| 1079 |
_view_title(datasets, metrics, dataset_id, metric_id)
|
| 1080 |
)
|
| 1081 |
-
metric_update = gr.update(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1082 |
views = _views(
|
| 1083 |
dataset_id,
|
| 1084 |
metric_id,
|
|
@@ -1090,7 +1302,14 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1090 |
num_prompts,
|
| 1091 |
seed,
|
| 1092 |
)
|
| 1093 |
-
return (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1094 |
|
| 1095 |
def on_models(
|
| 1096 |
dataset_id,
|
|
@@ -1104,8 +1323,19 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1104 |
seed,
|
| 1105 |
):
|
| 1106 |
model_choices = _model_choices(datasets, dataset_id)
|
| 1107 |
-
|
| 1108 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1109 |
views = _views(
|
| 1110 |
dataset_id,
|
| 1111 |
metric_id,
|
|
@@ -1117,7 +1347,12 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1117 |
num_prompts,
|
| 1118 |
seed,
|
| 1119 |
)
|
| 1120 |
-
return (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1121 |
|
| 1122 |
def on_leaderboard_filters(
|
| 1123 |
dataset_id,
|
|
@@ -1137,11 +1372,14 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1137 |
optimized_value or [],
|
| 1138 |
models=models,
|
| 1139 |
)
|
|
|
|
|
|
|
|
|
|
| 1140 |
return _leaderboard_html(
|
| 1141 |
filtered,
|
| 1142 |
view["columns"],
|
| 1143 |
-
|
| 1144 |
-
|
| 1145 |
)
|
| 1146 |
|
| 1147 |
def on_samples_controls(dataset_id, models, num_prompts, seed):
|
|
@@ -1163,23 +1401,19 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1163 |
next_seed,
|
| 1164 |
)
|
| 1165 |
|
| 1166 |
-
|
| 1167 |
-
|
| 1168 |
-
|
| 1169 |
-
|
| 1170 |
-
|
| 1171 |
-
owner,
|
| 1172 |
-
optimized,
|
| 1173 |
-
prompt_count,
|
| 1174 |
-
seed_state,
|
| 1175 |
]
|
|
|
|
| 1176 |
dataset_outputs = [
|
| 1177 |
lb_dataset,
|
| 1178 |
pp_dataset,
|
| 1179 |
sm_dataset,
|
| 1180 |
lb_metric,
|
| 1181 |
pp_metric,
|
| 1182 |
-
sm_metric,
|
| 1183 |
lb_models,
|
| 1184 |
pp_models,
|
| 1185 |
sm_models,
|
|
@@ -1191,37 +1425,53 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1191 |
optimized,
|
| 1192 |
lb_note,
|
| 1193 |
ranking,
|
| 1194 |
-
|
| 1195 |
-
time_plot,
|
| 1196 |
gallery,
|
|
|
|
| 1197 |
]
|
| 1198 |
-
for dataset_dd
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1199 |
dataset_dd.change(
|
| 1200 |
on_dataset,
|
| 1201 |
-
inputs=[
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1202 |
outputs=dataset_outputs,
|
| 1203 |
)
|
| 1204 |
|
| 1205 |
metric_outputs = [
|
| 1206 |
lb_metric,
|
| 1207 |
pp_metric,
|
| 1208 |
-
sm_metric,
|
| 1209 |
lb_title,
|
| 1210 |
pp_title,
|
| 1211 |
sm_title,
|
| 1212 |
lb_note,
|
| 1213 |
ranking,
|
| 1214 |
-
|
| 1215 |
-
time_plot,
|
| 1216 |
gallery,
|
|
|
|
| 1217 |
]
|
| 1218 |
-
for metric_dd in (
|
|
|
|
|
|
|
|
|
|
| 1219 |
metric_dd.change(
|
| 1220 |
on_metric,
|
| 1221 |
inputs=[
|
| 1222 |
-
|
| 1223 |
metric_dd,
|
| 1224 |
-
|
| 1225 |
search,
|
| 1226 |
platform,
|
| 1227 |
owner,
|
|
@@ -1238,16 +1488,20 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1238 |
sm_models,
|
| 1239 |
lb_note,
|
| 1240 |
ranking,
|
| 1241 |
-
|
| 1242 |
-
time_plot,
|
| 1243 |
gallery,
|
|
|
|
| 1244 |
]
|
| 1245 |
-
for models_dd in (
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1246 |
models_dd.change(
|
| 1247 |
on_models,
|
| 1248 |
inputs=[
|
| 1249 |
-
|
| 1250 |
-
|
| 1251 |
models_dd,
|
| 1252 |
search,
|
| 1253 |
platform,
|
|
@@ -1285,7 +1539,6 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
|
|
| 1285 |
outputs=[seed_state, gallery],
|
| 1286 |
)
|
| 1287 |
|
| 1288 |
-
|
| 1289 |
def render_about():
|
| 1290 |
with gr.Row(elem_classes="about-layout", equal_height=False):
|
| 1291 |
with gr.Column(scale=1, min_width=340, elem_classes="about-col"):
|
|
|
|
| 7 |
import pandas as pd
|
| 8 |
import plotly.graph_objects as go
|
| 9 |
|
| 10 |
+
from model_display import display_model_name
|
| 11 |
+
|
| 12 |
_MASCOT_PATH = Path(__file__).resolve().parent / "pruna-mascot.png"
|
| 13 |
_MASCOT_DATA_URI = (
|
| 14 |
"data:image/png;base64,"
|
|
|
|
| 21 |
MAX_COMPARE_MODELS = 4
|
| 22 |
DEFAULT_COMPARE_PROMPTS = 3
|
| 23 |
MAX_COMPARE_PROMPTS = 8
|
| 24 |
+
MAX_PARETO_METRICS = 8
|
| 25 |
|
| 26 |
ABOUT_OVERVIEW_CONTENT = """
|
| 27 |
# About P-Bench
|
|
|
|
| 46 |
2. It generates one image per prompt when the run succeeds. Not every model
|
| 47 |
has every prompt or every metric.
|
| 48 |
3. Quality is scored automatically (OneIG alignment, P-Judger) and, where
|
| 49 |
+
available, by human preference (Datapoint Elo, Rapidata Elo).
|
|
|
|
| 50 |
4. Price per image and generation time are joined from the evaluation table.
|
| 51 |
|
| 52 |
## Current datasets
|
|
|
|
| 61 |
general object prompts (100 prompts each). This is the alignment slice of
|
| 62 |
OneIG, not the full suite. Alignment Overall is the mean of the category
|
| 63 |
scores that exist for that row.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
"""
|
| 65 |
|
| 66 |
ABOUT_DETAILS_CONTENT = """
|
|
|
|
| 77 |
- **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.
|
| 78 |
Rapidata rejects prompts over 400 characters, so this Elo is on a subset
|
| 79 |
of each suite (see Setup). Rapidata is not a dataset.
|
|
|
|
|
|
|
| 80 |
- **Generation time**: median and minimum generation time in seconds, as
|
| 81 |
reported in the evaluation table. This is not a p95, and we do not state
|
| 82 |
warm vs cold or concurrent load.
|
|
|
|
| 182 |
return items[0] if items else None
|
| 183 |
|
| 184 |
|
| 185 |
+
def _dataset_choices(datasets, *, require_samples=False):
|
| 186 |
+
return [
|
| 187 |
+
(dataset["name"], dataset["id"])
|
| 188 |
+
for dataset in datasets
|
| 189 |
+
if not require_samples or dataset.get("samples")
|
| 190 |
+
]
|
| 191 |
+
|
| 192 |
+
|
| 193 |
+
def _coerce_sample_dataset(datasets, dataset_id):
|
| 194 |
+
"""Prefer datasets that have compare-sample artifacts."""
|
| 195 |
+
dataset = _item(datasets, dataset_id)
|
| 196 |
+
if dataset and dataset.get("samples"):
|
| 197 |
+
return dataset["id"]
|
| 198 |
+
for candidate in datasets:
|
| 199 |
+
if candidate.get("samples"):
|
| 200 |
+
return candidate["id"]
|
| 201 |
+
return dataset_id
|
| 202 |
|
| 203 |
|
| 204 |
def _metric_choices(datasets, metrics, dataset_id):
|
|
|
|
| 216 |
|
| 217 |
|
| 218 |
def _coerce_metric(datasets, metrics, dataset_id, metric_id):
|
| 219 |
+
"""Keep a selected metric only if it is valid for this dataset."""
|
| 220 |
+
if isinstance(metric_id, (list, tuple)):
|
| 221 |
+
metric_id = metric_id[0] if metric_id else None
|
| 222 |
+
if metric_id is None or metric_id == "":
|
| 223 |
+
return None
|
| 224 |
choices = _metric_choices(datasets, metrics, dataset_id)
|
| 225 |
ids = [choice[1] for choice in choices]
|
| 226 |
if metric_id in ids:
|
| 227 |
return metric_id
|
| 228 |
+
return None
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def _metric_dropdown_value(metric_id):
|
| 232 |
+
"""Gradio multiselect value for an optional single metric."""
|
| 233 |
+
return [metric_id] if metric_id else []
|
| 234 |
|
| 235 |
|
| 236 |
def _model_choices(datasets, dataset_id):
|
|
|
|
| 238 |
data = dataset.get("data") if dataset else None
|
| 239 |
if data is None or "Model" not in getattr(data, "columns", []):
|
| 240 |
return []
|
| 241 |
+
models = data["Model"].dropna().astype(str).unique().tolist()
|
| 242 |
+
# (label, value) so the UI shows the shared name but filters on the raw id.
|
| 243 |
+
return sorted(
|
| 244 |
+
((display_model_name(model), model) for model in models),
|
| 245 |
+
key=lambda item: item[0].casefold(),
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _model_choice_values(choices):
|
| 250 |
+
return [choice[1] if isinstance(choice, tuple) else choice for choice in choices]
|
| 251 |
+
|
| 252 |
+
|
| 253 |
+
def _metric_columns(datasets, metrics, dataset_id):
|
| 254 |
+
dataset = _item(datasets, dataset_id)
|
| 255 |
+
if not dataset:
|
| 256 |
+
return []
|
| 257 |
+
data = dataset.get("data")
|
| 258 |
+
columns = getattr(data, "columns", [])
|
| 259 |
+
allowed = set(dataset.get("metric_ids") or [])
|
| 260 |
+
return [
|
| 261 |
+
metric["column"]
|
| 262 |
+
for metric in metrics
|
| 263 |
+
if metric["id"] in allowed and metric["column"] in columns
|
| 264 |
+
]
|
| 265 |
|
| 266 |
|
| 267 |
def _view_title(datasets, metrics, dataset_id, metric_id):
|
| 268 |
dataset = _item(datasets, dataset_id)
|
|
|
|
| 269 |
dataset_name = dataset["name"] if dataset else "Dataset"
|
| 270 |
+
metric = _item(metrics, metric_id) if metric_id else None
|
| 271 |
+
if not metric:
|
| 272 |
+
return dataset_name
|
| 273 |
+
return f"{dataset_name} | {metric['name']}"
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
_LEADERBOARD_META_COLUMNS = [
|
| 277 |
+
"Median Generation Time (s)",
|
| 278 |
+
"Min Generation Time (s)",
|
| 279 |
+
"Price / Image (USD)",
|
| 280 |
+
"Evaluation Date (UTC)",
|
| 281 |
+
"Date",
|
| 282 |
+
]
|
| 283 |
|
| 284 |
|
| 285 |
def _columns_for_metric(dataset, metric_column):
|
| 286 |
+
"""When a metric is selected, show identity + that score + time/price."""
|
| 287 |
+
available = list(getattr(dataset.get("data"), "columns", [])) or list(
|
| 288 |
+
dataset.get("columns") or []
|
| 289 |
+
)
|
| 290 |
+
identity = [
|
| 291 |
+
column
|
| 292 |
+
for column in ["Model", "Platform", "Endpoint Owner", "Optimized"]
|
| 293 |
+
if column in available
|
| 294 |
+
]
|
| 295 |
+
meta = [column for column in _LEADERBOARD_META_COLUMNS if column in available]
|
| 296 |
+
if metric_column and metric_column in available:
|
| 297 |
+
return [*identity, metric_column, *meta]
|
| 298 |
+
return [column for column in (dataset.get("columns") or available) if column != "URL"]
|
| 299 |
|
| 300 |
|
| 301 |
def resolve_view(datasets, metrics, dataset_id, metric_id):
|
| 302 |
dataset = _item(datasets, dataset_id)
|
| 303 |
+
if not dataset:
|
|
|
|
|
|
|
| 304 |
return None
|
| 305 |
+
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 306 |
+
metric = _item(metrics, metric_id) if metric_id else None
|
| 307 |
+
score_column = metric["column"] if metric else None
|
| 308 |
+
if score_column:
|
| 309 |
+
columns = _columns_for_metric(dataset, score_column)
|
| 310 |
+
score_columns = [score_column]
|
| 311 |
+
else:
|
| 312 |
+
columns = [
|
| 313 |
+
column
|
| 314 |
+
for column in (dataset.get("columns") or [])
|
| 315 |
+
if column != "URL"
|
| 316 |
+
]
|
| 317 |
+
score_columns = _metric_columns(datasets, metrics, dataset_id)
|
| 318 |
return {
|
| 319 |
"dataset": dataset,
|
| 320 |
"metric": metric,
|
| 321 |
+
"metric_id": metric_id,
|
| 322 |
+
"title": _view_title(datasets, metrics, dataset["id"], metric_id),
|
| 323 |
"data": dataset["data"],
|
| 324 |
+
"columns": columns,
|
| 325 |
+
"score_column": score_column,
|
| 326 |
+
"score_columns": score_columns,
|
| 327 |
"samples": dataset.get("samples"),
|
| 328 |
"note": dataset.get("note"),
|
| 329 |
}
|
|
|
|
| 343 |
except (TypeError, ValueError):
|
| 344 |
return escape(str(value))
|
| 345 |
if label in {"model", "platform", "endpoint owner", "optimized"}:
|
| 346 |
+
if label == "model":
|
| 347 |
+
return escape(display_model_name(value))
|
| 348 |
return escape(str(value))
|
| 349 |
try:
|
| 350 |
number = float(value)
|
|
|
|
| 457 |
matches |= filtered[column].astype(str).str.contains(
|
| 458 |
search_term, case=False, na=False
|
| 459 |
)
|
| 460 |
+
if "Model" in filtered.columns:
|
| 461 |
+
display_names = filtered["Model"].map(display_model_name).astype(str)
|
| 462 |
+
matches |= display_names.str.contains(
|
| 463 |
+
search_term, case=False, na=False
|
| 464 |
+
)
|
| 465 |
filtered = filtered[matches]
|
| 466 |
|
| 467 |
for column, values in [
|
|
|
|
| 545 |
"P-Judge Overall": "P-Judger (Pruna)",
|
| 546 |
"Datapoint Elo": "Datapoint Elo",
|
| 547 |
"Rapidata Elo": "Rapidata Elo",
|
| 548 |
+
"Arena Elo": "Overall Elo",
|
| 549 |
+
"Arena Branding / Commercial Elo": "Branding / Commercial",
|
| 550 |
+
"Arena 3D Imaging Elo": "3D Imaging",
|
| 551 |
+
"Arena Cartoon / Anime Elo": "Cartoon / Anime",
|
| 552 |
+
"Arena Photorealistic Elo": "Photorealistic",
|
| 553 |
+
"Arena Art Elo": "Art",
|
| 554 |
+
"Arena Portraits Elo": "Portraits",
|
| 555 |
+
"Arena Text Rendering Elo": "Text Rendering",
|
| 556 |
"Raw Win Rate": "Raw win rate",
|
| 557 |
"Median Generation Time (s)": "Median generation time",
|
| 558 |
"Min Generation Time (s)": "Min generation time",
|
|
|
|
| 629 |
scores = scatter[score_column].astype(float).tolist()
|
| 630 |
on_frontier = _pareto_frontier_mask(x_values, scores)
|
| 631 |
|
| 632 |
+
dominated = scatter.loc[[not flag for flag in on_frontier]].copy()
|
| 633 |
+
frontier = scatter.loc[on_frontier].sort_values(x_column).copy()
|
| 634 |
+
if not dominated.empty:
|
| 635 |
+
dominated["Model"] = dominated["Model"].map(display_model_name)
|
| 636 |
+
if not frontier.empty:
|
| 637 |
+
frontier["Model"] = frontier["Model"].map(display_model_name)
|
| 638 |
hover = (
|
| 639 |
"<b>%{text}</b><br>"
|
| 640 |
f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"
|
|
|
|
| 750 |
)
|
| 751 |
|
| 752 |
|
| 753 |
+
def _pareto_slot_updates(data, score_columns):
|
| 754 |
+
"""Updates for a fixed bank of Gradio Plot slots (visible/hidden)."""
|
| 755 |
+
score_columns = [column for column in (score_columns or []) if column]
|
| 756 |
+
updates = []
|
| 757 |
+
for index in range(MAX_PARETO_METRICS):
|
| 758 |
+
if index < len(score_columns):
|
| 759 |
+
score_column = score_columns[index]
|
| 760 |
+
price_fig, time_fig = _pareto_pair(data, score_column)
|
| 761 |
+
updates.extend(
|
| 762 |
+
[
|
| 763 |
+
gr.update(visible=True),
|
| 764 |
+
f"#### {_display_label(score_column)}",
|
| 765 |
+
price_fig,
|
| 766 |
+
time_fig,
|
| 767 |
+
]
|
| 768 |
+
)
|
| 769 |
+
else:
|
| 770 |
+
updates.extend(
|
| 771 |
+
[
|
| 772 |
+
gr.update(visible=False),
|
| 773 |
+
"",
|
| 774 |
+
None,
|
| 775 |
+
None,
|
| 776 |
+
]
|
| 777 |
+
)
|
| 778 |
+
return updates
|
| 779 |
+
|
| 780 |
+
|
| 781 |
def _samples_html(samples, selected_models, num_prompts, seed=0):
|
| 782 |
if not samples:
|
| 783 |
+
return ""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 784 |
models = list(selected_models or [])
|
| 785 |
available = samples.get("models") or []
|
| 786 |
models = [model for model in models if model in samples.get("images", {})]
|
|
|
|
| 835 |
cells.append(
|
| 836 |
f"""
|
| 837 |
<div class="compare-cell">
|
| 838 |
+
<div class="compare-model-label">{escape(display_model_name(model))}</div>
|
| 839 |
<a href="{image_url}" target="_blank" rel="noopener noreferrer">
|
| 840 |
+
<img src="{image_url}" alt="{escape(display_model_name(model))} sample" loading="lazy" />
|
| 841 |
</a>
|
| 842 |
</div>
|
| 843 |
"""
|
|
|
|
| 868 |
return note or ""
|
| 869 |
|
| 870 |
|
| 871 |
+
def _filter_row(
|
| 872 |
+
datasets,
|
| 873 |
+
metrics,
|
| 874 |
+
default_dataset_id,
|
| 875 |
+
default_metric_id=None,
|
| 876 |
+
*,
|
| 877 |
+
require_samples=False,
|
| 878 |
+
include_metric=True,
|
| 879 |
+
):
|
| 880 |
metric_choices = _metric_choices(datasets, metrics, default_dataset_id)
|
| 881 |
model_choices = _model_choices(datasets, default_dataset_id)
|
| 882 |
+
metric_id = _coerce_metric(
|
| 883 |
+
datasets, metrics, default_dataset_id, default_metric_id
|
| 884 |
+
)
|
| 885 |
with gr.Row(elem_classes="view-filters"):
|
| 886 |
dataset_dd = gr.Dropdown(
|
| 887 |
+
choices=_dataset_choices(datasets, require_samples=require_samples),
|
| 888 |
value=default_dataset_id,
|
| 889 |
label="Dataset",
|
| 890 |
type="value",
|
| 891 |
scale=2,
|
| 892 |
min_width=160,
|
| 893 |
)
|
| 894 |
+
metric_dd = None
|
| 895 |
+
if include_metric:
|
| 896 |
+
metric_dd = gr.Dropdown(
|
| 897 |
+
choices=metric_choices,
|
| 898 |
+
value=_metric_dropdown_value(metric_id),
|
| 899 |
+
label="Metric",
|
| 900 |
+
type="value",
|
| 901 |
+
multiselect=True,
|
| 902 |
+
max_choices=1,
|
| 903 |
+
allow_custom_value=True,
|
| 904 |
+
scale=2,
|
| 905 |
+
min_width=180,
|
| 906 |
+
)
|
| 907 |
models_dd = gr.Dropdown(
|
| 908 |
choices=model_choices,
|
| 909 |
value=[],
|
| 910 |
multiselect=True,
|
| 911 |
label="Models",
|
| 912 |
type="value",
|
| 913 |
+
allow_custom_value=True,
|
| 914 |
scale=3,
|
| 915 |
min_width=200,
|
| 916 |
)
|
| 917 |
title = gr.Markdown(
|
| 918 |
_title_markdown(
|
| 919 |
+
_view_title(datasets, metrics, default_dataset_id, metric_id)
|
| 920 |
),
|
| 921 |
elem_classes="view-title",
|
| 922 |
)
|
|
|
|
| 930 |
initial = resolve_view(datasets, metrics, default_dataset_id, default_metric_id)
|
| 931 |
initial_data = initial["data"]
|
| 932 |
initial_columns = initial["columns"]
|
| 933 |
+
initial_score_columns = initial["score_columns"]
|
| 934 |
initial_samples = initial.get("samples")
|
| 935 |
+
initial_pareto_updates = _pareto_slot_updates(
|
| 936 |
+
_filter_leaderboard(initial_data, "", [], [], [], models=[]),
|
| 937 |
+
initial_score_columns,
|
| 938 |
+
)
|
| 939 |
|
| 940 |
with gr.Tabs(elem_classes="main-tabs"):
|
| 941 |
with gr.TabItem("Leaderboards"):
|
| 942 |
lb_dataset, lb_metric, lb_models, lb_title = _filter_row(
|
| 943 |
+
datasets, metrics, default_dataset_id, None
|
| 944 |
)
|
| 945 |
lb_note = gr.Markdown(_note_markdown(initial.get("note")))
|
| 946 |
platform_choices = _filter_choices(initial_data, "Platform")
|
|
|
|
| 980 |
)
|
| 981 |
ranking = gr.HTML(
|
| 982 |
_leaderboard_html(
|
| 983 |
+
initial_data,
|
| 984 |
+
initial_columns,
|
| 985 |
+
initial_score_columns,
|
| 986 |
+
initial_score_columns[0] if initial_score_columns else None,
|
| 987 |
),
|
| 988 |
padding=False,
|
| 989 |
elem_classes="ranking-table-host",
|
|
|
|
| 991 |
|
| 992 |
with gr.TabItem("Pareto Plots"):
|
| 993 |
pp_dataset, pp_metric, pp_models, pp_title = _filter_row(
|
| 994 |
+
datasets, metrics, default_dataset_id, None
|
| 995 |
)
|
| 996 |
gr.Markdown(
|
| 997 |
"<span class='pareto-help'>"
|
| 998 |
"Green = on the frontier (lower cost or time at the same or better score). "
|
| 999 |
+
"Lavender = below the frontier. Pick a metric to focus on one pair of plots; "
|
| 1000 |
+
"leave it empty to see every metric."
|
| 1001 |
"</span><br/>"
|
| 1002 |
"<strong class='pareto-help-emphasis'>Hover a point to see which model it is.</strong>"
|
| 1003 |
)
|
| 1004 |
+
pareto_slots = []
|
| 1005 |
+
for slot_index in range(MAX_PARETO_METRICS):
|
| 1006 |
+
offset = slot_index * 4
|
| 1007 |
+
visible = slot_index < len(initial_score_columns)
|
| 1008 |
+
with gr.Column(
|
| 1009 |
+
visible=visible,
|
| 1010 |
+
elem_classes="pareto-metric-block",
|
| 1011 |
+
) as slot_group:
|
| 1012 |
+
slot_title = gr.Markdown(
|
| 1013 |
+
(
|
| 1014 |
+
f"#### {_display_label(initial_score_columns[slot_index])}"
|
| 1015 |
+
if visible
|
| 1016 |
+
else ""
|
| 1017 |
+
),
|
| 1018 |
+
elem_classes="pareto-metric-title",
|
| 1019 |
)
|
| 1020 |
+
with gr.Row(equal_height=False, elem_classes="pareto-layout"):
|
| 1021 |
+
with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
|
| 1022 |
+
gr.Markdown("#### Price vs score")
|
| 1023 |
+
slot_price = gr.Plot(
|
| 1024 |
+
value=(
|
| 1025 |
+
initial_pareto_updates[offset + 2]
|
| 1026 |
+
if visible
|
| 1027 |
+
else None
|
| 1028 |
+
),
|
| 1029 |
+
show_label=False,
|
| 1030 |
+
elem_classes="pareto-plot",
|
| 1031 |
+
)
|
| 1032 |
+
with gr.Column(scale=1, min_width=320, elem_classes="pareto-col"):
|
| 1033 |
+
gr.Markdown("#### Min generation time vs score")
|
| 1034 |
+
slot_time = gr.Plot(
|
| 1035 |
+
value=(
|
| 1036 |
+
initial_pareto_updates[offset + 3]
|
| 1037 |
+
if visible
|
| 1038 |
+
else None
|
| 1039 |
+
),
|
| 1040 |
+
show_label=False,
|
| 1041 |
+
elem_classes="pareto-plot",
|
| 1042 |
+
)
|
| 1043 |
+
pareto_slots.append(
|
| 1044 |
+
(slot_group, slot_title, slot_price, slot_time)
|
| 1045 |
+
)
|
| 1046 |
|
| 1047 |
with gr.TabItem("Samples"):
|
| 1048 |
+
sample_default_dataset_id = _coerce_sample_dataset(
|
| 1049 |
+
datasets, default_dataset_id
|
| 1050 |
)
|
| 1051 |
+
sm_dataset, _, sm_models, sm_title = _filter_row(
|
| 1052 |
+
datasets,
|
| 1053 |
+
metrics,
|
| 1054 |
+
sample_default_dataset_id,
|
| 1055 |
+
None,
|
| 1056 |
+
require_samples=True,
|
| 1057 |
+
include_metric=False,
|
|
|
|
|
|
|
| 1058 |
)
|
| 1059 |
+
with gr.Column(visible=bool(initial_samples)) as samples_panel:
|
| 1060 |
+
gr.Markdown(
|
| 1061 |
+
f"""
|
| 1062 |
+
<p class="compare-samples-help">
|
| 1063 |
+
Filter models above (up to <strong>{MAX_COMPARE_MODELS}</strong> are
|
| 1064 |
+
shown). If none are selected, two defaults appear. Images come from
|
| 1065 |
+
the public generation URLs for this dataset. Prompts to show chooses
|
| 1066 |
+
how many shared prompts appear (1–{MAX_COMPARE_PROMPTS}).
|
| 1067 |
+
</p>
|
| 1068 |
+
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1069 |
)
|
| 1070 |
+
with gr.Row(equal_height=False, elem_classes="compare-controls"):
|
| 1071 |
+
prompt_count = gr.Slider(
|
| 1072 |
+
minimum=1,
|
| 1073 |
+
maximum=MAX_COMPARE_PROMPTS,
|
| 1074 |
+
value=DEFAULT_COMPARE_PROMPTS,
|
| 1075 |
+
step=1,
|
| 1076 |
+
label="Prompts to show",
|
| 1077 |
+
container=False,
|
| 1078 |
+
show_reset_button=False,
|
| 1079 |
+
scale=1,
|
| 1080 |
+
min_width=180,
|
| 1081 |
+
elem_classes="compare-prompt-count",
|
| 1082 |
+
)
|
| 1083 |
+
shuffle_button = gr.Button(
|
| 1084 |
+
"Shuffle prompts",
|
| 1085 |
+
variant="primary",
|
| 1086 |
+
scale=0,
|
| 1087 |
+
min_width=140,
|
| 1088 |
+
elem_classes="compare-shuffle",
|
| 1089 |
+
)
|
| 1090 |
gallery = gr.HTML(
|
| 1091 |
value=_samples_html(
|
| 1092 |
initial_samples, [], DEFAULT_COMPARE_PROMPTS, seed=0
|
|
|
|
| 1098 |
with gr.TabItem("About"):
|
| 1099 |
render_about()
|
| 1100 |
|
| 1101 |
+
def _synced_filters(dataset_id, metric_id, models, *, clear_metric=False):
|
| 1102 |
+
if clear_metric:
|
| 1103 |
+
metric_id = None
|
| 1104 |
+
else:
|
| 1105 |
+
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 1106 |
model_choices = _model_choices(datasets, dataset_id)
|
| 1107 |
+
model_values = set(_model_choice_values(model_choices))
|
| 1108 |
+
models = [model for model in (models or []) if model in model_values]
|
| 1109 |
metric_choices = _metric_choices(datasets, metrics, dataset_id)
|
| 1110 |
title = _title_markdown(
|
| 1111 |
_view_title(datasets, metrics, dataset_id, metric_id)
|
| 1112 |
)
|
| 1113 |
dataset_update = gr.update(value=dataset_id)
|
| 1114 |
+
metric_update = gr.update(
|
| 1115 |
+
choices=metric_choices, value=_metric_dropdown_value(metric_id)
|
| 1116 |
+
)
|
| 1117 |
models_update = gr.update(choices=model_choices, value=models)
|
| 1118 |
+
|
| 1119 |
+
sample_dataset_id = _coerce_sample_dataset(datasets, dataset_id)
|
| 1120 |
+
sample_model_choices = _model_choices(datasets, sample_dataset_id)
|
| 1121 |
+
sample_model_values = set(_model_choice_values(sample_model_choices))
|
| 1122 |
+
sample_models = [
|
| 1123 |
+
model for model in (models or []) if model in sample_model_values
|
| 1124 |
+
]
|
| 1125 |
+
sample_title = _title_markdown(
|
| 1126 |
+
_view_title(datasets, metrics, sample_dataset_id, None)
|
| 1127 |
+
)
|
| 1128 |
return (
|
| 1129 |
dataset_id,
|
| 1130 |
metric_id,
|
| 1131 |
models,
|
| 1132 |
+
sample_dataset_id,
|
| 1133 |
dataset_update,
|
| 1134 |
dataset_update,
|
| 1135 |
+
gr.update(value=sample_dataset_id),
|
| 1136 |
metric_update,
|
| 1137 |
metric_update,
|
|
|
|
|
|
|
| 1138 |
models_update,
|
| 1139 |
models_update,
|
| 1140 |
+
gr.update(choices=sample_model_choices, value=sample_models),
|
| 1141 |
title,
|
| 1142 |
title,
|
| 1143 |
+
sample_title,
|
| 1144 |
)
|
| 1145 |
|
| 1146 |
def _leaderboard_extras(data, platform_value, owner_value, optimized_value):
|
|
|
|
| 1198 |
optimized_value or [],
|
| 1199 |
models=models,
|
| 1200 |
)
|
| 1201 |
+
sort_column = view["score_column"] or (
|
| 1202 |
+
view["score_columns"][0] if view["score_columns"] else None
|
| 1203 |
+
)
|
| 1204 |
ranking_html = _leaderboard_html(
|
| 1205 |
filtered,
|
| 1206 |
view["columns"],
|
| 1207 |
+
view["score_columns"],
|
| 1208 |
+
sort_column,
|
| 1209 |
)
|
| 1210 |
+
pareto_data = _filter_leaderboard(data, "", [], [], [], models=models)
|
| 1211 |
+
pareto_updates = _pareto_slot_updates(pareto_data, view["score_columns"])
|
| 1212 |
+
sample_dataset_id = _coerce_sample_dataset(datasets, dataset_id)
|
| 1213 |
+
sample_view = resolve_view(datasets, metrics, sample_dataset_id, None)
|
| 1214 |
+
samples = sample_view.get("samples") if sample_view else None
|
| 1215 |
+
sample_models = [
|
| 1216 |
+
model
|
| 1217 |
+
for model in (models or [])
|
| 1218 |
+
if model
|
| 1219 |
+
in _model_choice_values(_model_choices(datasets, sample_dataset_id))
|
| 1220 |
+
]
|
| 1221 |
samples_html = _samples_html(
|
| 1222 |
+
samples,
|
| 1223 |
+
sample_models,
|
| 1224 |
int(num_prompts or DEFAULT_COMPARE_PROMPTS),
|
| 1225 |
int(seed or 0),
|
| 1226 |
)
|
| 1227 |
return (
|
| 1228 |
_note_markdown(view.get("note")),
|
| 1229 |
ranking_html,
|
| 1230 |
+
*pareto_updates,
|
|
|
|
| 1231 |
samples_html,
|
| 1232 |
+
gr.update(visible=bool(samples)),
|
| 1233 |
)
|
| 1234 |
|
| 1235 |
def on_dataset(
|
|
|
|
| 1243 |
num_prompts,
|
| 1244 |
seed,
|
| 1245 |
):
|
| 1246 |
+
synced = _synced_filters(
|
| 1247 |
+
dataset_id, metric_id, models, clear_metric=True
|
| 1248 |
+
)
|
| 1249 |
dataset_id, metric_id, models = synced[:3]
|
| 1250 |
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1251 |
extras = _leaderboard_extras(
|
|
|
|
| 1262 |
num_prompts,
|
| 1263 |
seed,
|
| 1264 |
)
|
| 1265 |
+
return (*synced[4:], extras[0], extras[1], extras[2], *views)
|
| 1266 |
|
| 1267 |
def on_metric(
|
| 1268 |
dataset_id,
|
|
|
|
| 1276 |
seed,
|
| 1277 |
):
|
| 1278 |
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 1279 |
+
metric_choices = _metric_choices(datasets, metrics, dataset_id)
|
| 1280 |
title = _title_markdown(
|
| 1281 |
_view_title(datasets, metrics, dataset_id, metric_id)
|
| 1282 |
)
|
| 1283 |
+
metric_update = gr.update(
|
| 1284 |
+
choices=metric_choices, value=_metric_dropdown_value(metric_id)
|
| 1285 |
+
)
|
| 1286 |
+
sample_title = _title_markdown(
|
| 1287 |
+
_view_title(
|
| 1288 |
+
datasets,
|
| 1289 |
+
metrics,
|
| 1290 |
+
_coerce_sample_dataset(datasets, dataset_id),
|
| 1291 |
+
None,
|
| 1292 |
+
)
|
| 1293 |
+
)
|
| 1294 |
views = _views(
|
| 1295 |
dataset_id,
|
| 1296 |
metric_id,
|
|
|
|
| 1302 |
num_prompts,
|
| 1303 |
seed,
|
| 1304 |
)
|
| 1305 |
+
return (
|
| 1306 |
+
metric_update,
|
| 1307 |
+
metric_update,
|
| 1308 |
+
title,
|
| 1309 |
+
title,
|
| 1310 |
+
sample_title,
|
| 1311 |
+
*views,
|
| 1312 |
+
)
|
| 1313 |
|
| 1314 |
def on_models(
|
| 1315 |
dataset_id,
|
|
|
|
| 1323 |
seed,
|
| 1324 |
):
|
| 1325 |
model_choices = _model_choices(datasets, dataset_id)
|
| 1326 |
+
model_values = set(_model_choice_values(model_choices))
|
| 1327 |
+
models = [model for model in (models or []) if model in model_values]
|
| 1328 |
+
models_update = gr.update(choices=model_choices, value=models)
|
| 1329 |
+
|
| 1330 |
+
sample_dataset_id = _coerce_sample_dataset(datasets, dataset_id)
|
| 1331 |
+
sample_model_choices = _model_choices(datasets, sample_dataset_id)
|
| 1332 |
+
sample_model_values = set(_model_choice_values(sample_model_choices))
|
| 1333 |
+
sample_models = [
|
| 1334 |
+
model for model in (models or []) if model in sample_model_values
|
| 1335 |
+
]
|
| 1336 |
+
sample_models_update = gr.update(
|
| 1337 |
+
choices=sample_model_choices, value=sample_models
|
| 1338 |
+
)
|
| 1339 |
views = _views(
|
| 1340 |
dataset_id,
|
| 1341 |
metric_id,
|
|
|
|
| 1347 |
num_prompts,
|
| 1348 |
seed,
|
| 1349 |
)
|
| 1350 |
+
return (
|
| 1351 |
+
models_update,
|
| 1352 |
+
models_update,
|
| 1353 |
+
sample_models_update,
|
| 1354 |
+
*views,
|
| 1355 |
+
)
|
| 1356 |
|
| 1357 |
def on_leaderboard_filters(
|
| 1358 |
dataset_id,
|
|
|
|
| 1372 |
optimized_value or [],
|
| 1373 |
models=models,
|
| 1374 |
)
|
| 1375 |
+
sort_column = view["score_column"] or (
|
| 1376 |
+
view["score_columns"][0] if view["score_columns"] else None
|
| 1377 |
+
)
|
| 1378 |
return _leaderboard_html(
|
| 1379 |
filtered,
|
| 1380 |
view["columns"],
|
| 1381 |
+
view["score_columns"],
|
| 1382 |
+
sort_column,
|
| 1383 |
)
|
| 1384 |
|
| 1385 |
def on_samples_controls(dataset_id, models, num_prompts, seed):
|
|
|
|
| 1401 |
next_seed,
|
| 1402 |
)
|
| 1403 |
|
| 1404 |
+
no_metric = gr.State(None)
|
| 1405 |
+
pareto_outputs = [
|
| 1406 |
+
component
|
| 1407 |
+
for slot_group, slot_title, slot_price, slot_time in pareto_slots
|
| 1408 |
+
for component in (slot_group, slot_title, slot_price, slot_time)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1409 |
]
|
| 1410 |
+
|
| 1411 |
dataset_outputs = [
|
| 1412 |
lb_dataset,
|
| 1413 |
pp_dataset,
|
| 1414 |
sm_dataset,
|
| 1415 |
lb_metric,
|
| 1416 |
pp_metric,
|
|
|
|
| 1417 |
lb_models,
|
| 1418 |
pp_models,
|
| 1419 |
sm_models,
|
|
|
|
| 1425 |
optimized,
|
| 1426 |
lb_note,
|
| 1427 |
ranking,
|
| 1428 |
+
*pareto_outputs,
|
|
|
|
| 1429 |
gallery,
|
| 1430 |
+
samples_panel,
|
| 1431 |
]
|
| 1432 |
+
for dataset_dd, metric_dd, models_dd in (
|
| 1433 |
+
(lb_dataset, lb_metric, lb_models),
|
| 1434 |
+
(pp_dataset, pp_metric, pp_models),
|
| 1435 |
+
(sm_dataset, no_metric, sm_models),
|
| 1436 |
+
):
|
| 1437 |
dataset_dd.change(
|
| 1438 |
on_dataset,
|
| 1439 |
+
inputs=[
|
| 1440 |
+
dataset_dd,
|
| 1441 |
+
metric_dd,
|
| 1442 |
+
models_dd,
|
| 1443 |
+
search,
|
| 1444 |
+
platform,
|
| 1445 |
+
owner,
|
| 1446 |
+
optimized,
|
| 1447 |
+
prompt_count,
|
| 1448 |
+
seed_state,
|
| 1449 |
+
],
|
| 1450 |
outputs=dataset_outputs,
|
| 1451 |
)
|
| 1452 |
|
| 1453 |
metric_outputs = [
|
| 1454 |
lb_metric,
|
| 1455 |
pp_metric,
|
|
|
|
| 1456 |
lb_title,
|
| 1457 |
pp_title,
|
| 1458 |
sm_title,
|
| 1459 |
lb_note,
|
| 1460 |
ranking,
|
| 1461 |
+
*pareto_outputs,
|
|
|
|
| 1462 |
gallery,
|
| 1463 |
+
samples_panel,
|
| 1464 |
]
|
| 1465 |
+
for dataset_dd, metric_dd, models_dd in (
|
| 1466 |
+
(lb_dataset, lb_metric, lb_models),
|
| 1467 |
+
(pp_dataset, pp_metric, pp_models),
|
| 1468 |
+
):
|
| 1469 |
metric_dd.change(
|
| 1470 |
on_metric,
|
| 1471 |
inputs=[
|
| 1472 |
+
dataset_dd,
|
| 1473 |
metric_dd,
|
| 1474 |
+
models_dd,
|
| 1475 |
search,
|
| 1476 |
platform,
|
| 1477 |
owner,
|
|
|
|
| 1488 |
sm_models,
|
| 1489 |
lb_note,
|
| 1490 |
ranking,
|
| 1491 |
+
*pareto_outputs,
|
|
|
|
| 1492 |
gallery,
|
| 1493 |
+
samples_panel,
|
| 1494 |
]
|
| 1495 |
+
for dataset_dd, metric_dd, models_dd in (
|
| 1496 |
+
(lb_dataset, lb_metric, lb_models),
|
| 1497 |
+
(pp_dataset, pp_metric, pp_models),
|
| 1498 |
+
(sm_dataset, no_metric, sm_models),
|
| 1499 |
+
):
|
| 1500 |
models_dd.change(
|
| 1501 |
on_models,
|
| 1502 |
inputs=[
|
| 1503 |
+
dataset_dd,
|
| 1504 |
+
metric_dd,
|
| 1505 |
models_dd,
|
| 1506 |
search,
|
| 1507 |
platform,
|
|
|
|
| 1539 |
outputs=[seed_state, gallery],
|
| 1540 |
)
|
| 1541 |
|
|
|
|
| 1542 |
def render_about():
|
| 1543 |
with gr.Row(elem_classes="about-layout", equal_height=False):
|
| 1544 |
with gr.Column(scale=1, min_width=340, elem_classes="about-col"):
|