Minette Kaunismäki commited on
Commit
a9c18ff
·
1 Parent(s): 335823a

cleaning and updating about

Browse files
Files changed (5) hide show
  1. README.md +15 -54
  2. app.py +61 -199
  3. model_display.py +0 -37
  4. pruna-mascot.png → pruna-logo.png +0 -0
  5. ui.py +298 -333
README.md CHANGED
@@ -14,71 +14,32 @@ tags:
14
 
15
  # P-Bench
16
 
17
- Evaluate the quality and efficiency of image gen api's.
 
18
 
19
- ## Installation
20
 
21
- ### Install dependencies
22
 
23
- Install dependencies with conda like that:
24
- ```
25
- conda env create -f environment.yml
26
- ```
27
-
28
- ### Install uv
29
-
30
- Install uv with pip like that:
31
-
32
- ```
33
- uv venv --python 3.12
34
- ```
35
-
36
- Then activate the environment:
37
 
38
  ```
 
39
  source .venv/bin/activate
 
 
40
  ```
41
 
42
- Then install the dependencies with uv:
43
 
44
- ```
45
- uv sync --all-groups
46
- ```
47
 
48
- ## Usage
49
 
50
- Create .env file with all the credentials you will need.
 
51
 
52
- This is how you can generate the images.
53
- ```
54
- python sample.py replicate draw_bench genai_bench geneval hps parti
55
- ```
56
-
57
- This is how you would evaluate the benchmarks once you have all images:
58
- ```
59
- python evaluate.py replicate draw_bench genai_bench geneval hps parti
60
  ```
61
-
62
- ## Leaderboard
63
-
64
- The leaderboard is [hosted on Hugging Face](https://huggingface.co/spaces/PrunaAI/InferBench/tree/main) with gradio.
65
-
66
- To run the dashboard locally, you can use the following command:
67
-
68
- ```
69
- python dashboard/app.py
70
- ```
71
-
72
- To deploy the dashboard to Hugging Face, you can use the following commands:
73
-
74
- First, add the remote:
75
-
76
- ```
77
- git remote add hf https://huggingface.co/spaces/PrunaAI/InferBench
78
- ```
79
-
80
- Then push the changes of your branch to the remote:
81
-
82
  ```
83
- git push hf $(git rev-parse --abbrev-ref HEAD):main --force
84
- ```
 
14
 
15
  # P-Bench
16
 
17
+ Compare text-to-image models on quality, speed, and price. This repo is the
18
+ Gradio dashboard: leaderboards, Pareto plots, and side-by-side samples.
19
 
20
+ The live Space is [PrunaAI/P-Bench](https://huggingface.co/spaces/PrunaAI/P-Bench).
21
 
22
+ ## Run locally
23
 
24
+ From the repo root:
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
  ```
27
+ python -m venv .venv
28
  source .venv/bin/activate
29
+ pip install "gradio==5.19.0" pandas -r requirements.txt
30
+ python app.py
31
  ```
32
 
33
+ The app is served at `http://127.0.0.1:7860`.
34
 
35
+ `requirements.txt` lists Plotly. Gradio and pandas are required locally;
36
+ Hugging Face Spaces installs Gradio from the YAML `sdk_version` above.
 
37
 
38
+ ## Deploy
39
 
40
+ `origin` is the Space (`https://huggingface.co/spaces/PrunaAI/P-Bench`).
41
+ Publish the current branch to the live app with:
42
 
 
 
 
 
 
 
 
 
43
  ```
44
+ git push origin HEAD:main
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  ```
 
 
app.py CHANGED
@@ -1914,18 +1914,11 @@ def load_sample_comparison_data(folder):
1914
  }
1915
 
1916
 
1917
- def format_url_links(value):
1918
- if pd.isna(value) or not value:
1919
- return ""
1920
-
1921
- links = [url.strip() for url in str(value).split(",") if url.strip()]
1922
- if not links:
1923
- return ""
1924
-
1925
- return " ".join(
1926
- f'<a target="_blank" href="{url}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">run {idx}</a>'
1927
- for idx, url in enumerate(links, start=1)
1928
- )
1929
 
1930
 
1931
  def load_oneig_dataframe(path):
@@ -1933,7 +1926,6 @@ def load_oneig_dataframe(path):
1933
  df = df.rename(
1934
  columns={
1935
  "Owner": "Endpoint Owner",
1936
- "Optimization": "Optimization Details",
1937
  "Anime Alignment Score": "OneIG (Anime Alignment)",
1938
  "Human Alignment Score": "OneIG (Human Alignment)",
1939
  "Object Alignment Score": "OneIG (Object Alignment)",
@@ -1942,63 +1934,34 @@ def load_oneig_dataframe(path):
1942
  "OneIG (General Object) (Alignment Score)": "OneIG (Object Alignment)",
1943
  }
1944
  )
1945
- if "URL" in df.columns:
1946
- df["URL"] = df["URL"].apply(format_url_links)
 
 
 
 
 
1947
  if "Optimized" in df.columns:
1948
  df["Optimized"] = df["Optimized"].map(
1949
  {True: "Yes", False: "No", "TRUE": "Yes", "FALSE": "No"}
1950
  ).fillna(df["Optimized"])
1951
 
1952
- for column in [
1953
- "Price / Image (USD)",
1954
- "Median Generation Time (s)",
1955
- "Min Generation Time (s)",
1956
- "OneIG (Anime Alignment)",
1957
- "OneIG (Human Alignment)",
1958
- "OneIG (Object Alignment)",
1959
- "OneIG Anime Elo",
1960
- "OneIG Human Elo",
1961
- "OneIG Object Elo",
1962
- "P-Judge Overall",
1963
- "Rapidata Elo",
1964
- ]:
1965
- if column in df.columns:
1966
- df[column] = pd.to_numeric(df[column], errors="coerce")
1967
-
1968
- preferred_columns = [
1969
- "Platform",
1970
- "Endpoint Owner",
1971
- "Device",
1972
- "Model",
1973
- "Optimized",
1974
- "Optimization Details",
1975
- "OneIG (Anime Alignment)",
1976
- "OneIG (Human Alignment)",
1977
- "OneIG (Object Alignment)",
1978
- "OneIG Anime Elo",
1979
- "OneIG Human Elo",
1980
- "OneIG Object Elo",
1981
- "Median Generation Time (s)",
1982
- "Min Generation Time (s)",
1983
- "Price / Image (USD)",
1984
- "Evaluation Date (UTC)",
1985
- "URL",
1986
- ]
1987
- present_preferred_columns = [col for col in preferred_columns if col in df.columns]
1988
- df = df[
1989
- present_preferred_columns
1990
- + [col for col in df.columns.tolist() if col not in present_preferred_columns]
1991
- ]
1992
-
1993
- if "OneIG (Human Alignment)" in df.columns:
1994
- df = df.sort_values(
1995
- by="OneIG (Human Alignment)", ascending=False, na_position="last"
1996
- )
1997
-
1998
- numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
1999
- for col in numeric_cols:
2000
- df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
2001
- return df
2002
 
2003
 
2004
  def load_artificial_analysis_dataframe(path):
@@ -2012,17 +1975,9 @@ def load_artificial_analysis_dataframe(path):
2012
  }
2013
  )
2014
  df["Model"] = df["Model"].astype(str).str.strip()
2015
- for column in ["Artificial Analysis Elo", "Price / Image (USD)"]:
2016
- if column in df.columns:
2017
- df[column] = pd.to_numeric(df[column], errors="coerce")
2018
- if "Artificial Analysis Elo" in df.columns:
2019
- df = df.sort_values(
2020
- by="Artificial Analysis Elo", ascending=False, na_position="last"
2021
- )
2022
- numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
2023
- for col in numeric_cols:
2024
- df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
2025
- return df.reset_index(drop=True)
2026
 
2027
 
2028
  ARENA_CATEGORY_COLUMNS = {
@@ -2054,13 +2009,7 @@ def load_arena_ai_dataframe(path):
2054
  for column in ["Arena Elo", *ARENA_CATEGORY_COLUMNS.values()]
2055
  if column in df.columns
2056
  ]
2057
- for column in score_columns:
2058
- df[column] = pd.to_numeric(df[column], errors="coerce")
2059
- if "Arena Elo" in df.columns:
2060
- df = df.sort_values(by="Arena Elo", ascending=False, na_position="last")
2061
- numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
2062
- for col in numeric_cols:
2063
- df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
2064
  ordered = ["Model", *score_columns]
2065
  return df[[column for column in ordered if column in df.columns]].reset_index(
2066
  drop=True
@@ -2077,36 +2026,17 @@ def load_qwen_combined_dataframe(path):
2077
  df = df[~df["Model"].astype(str).str.startswith("#")].copy()
2078
  df["Model"] = df["Model"].astype(str).str.strip()
2079
 
2080
- for column in [
2081
- "Price / Image (USD)",
2082
- "Median Generation Time (s)",
2083
- "Min Generation Time (s)",
2084
- "P-Judge Overall",
2085
- "Rapidata Elo",
2086
- "Datapoint Elo",
2087
- ]:
2088
- if column in df.columns:
2089
- df[column] = pd.to_numeric(df[column], errors="coerce")
2090
-
2091
- sort_column = next(
2092
- (
2093
- col
2094
- for col in [
2095
- "Datapoint Elo",
2096
- "P-Judge Overall",
2097
- "Rapidata Elo",
2098
- ]
2099
- if col in df.columns
2100
- ),
2101
- None,
2102
- )
2103
- if sort_column:
2104
- df = df.sort_values(by=sort_column, ascending=False, na_position="last")
2105
-
2106
- numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
2107
- for col in numeric_cols:
2108
- df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
2109
- return df.reset_index(drop=True)
2110
 
2111
 
2112
  df = load_oneig_dataframe(oneig_path)
@@ -2125,9 +2055,6 @@ if oneig_metric_columns:
2125
  oneig_df["OneIG Overall Score"] = oneig_df[oneig_metric_columns].mean(
2126
  axis=1, skipna=True
2127
  )
2128
- oneig_df = oneig_df.sort_values(
2129
- by="OneIG Overall Score", ascending=False, na_position="last"
2130
- )
2131
 
2132
  oneig_display_columns = [
2133
  col
@@ -2147,7 +2074,6 @@ oneig_display_columns = [
2147
  "Median Generation Time (s)",
2148
  "Min Generation Time (s)",
2149
  "Price / Image (USD)",
2150
- "URL",
2151
  ]
2152
  if col in oneig_df.columns
2153
  ]
@@ -2213,86 +2139,22 @@ oneig_samples = load_sample_comparison_data(oneig_combined_dir)
2213
  qwen_samples = load_sample_comparison_data(qwen_combined_dir)
2214
 
2215
  metrics = [
2216
- {
2217
- "id": "datapoint_elo",
2218
- "name": "Datapoint ELO - Overall Metric",
2219
- "column": "Datapoint Elo",
2220
- },
2221
- {
2222
- "id": "rapidata_elo",
2223
- "name": "Rapidata ELO - Overall Metric",
2224
- "column": "Rapidata Elo",
2225
- },
2226
- {
2227
- "id": "pjudger",
2228
- "name": "P-Judger Overall Metric",
2229
- "column": "P-Judge Overall",
2230
- },
2231
- {
2232
- "id": "alignment_overall",
2233
- "name": "Alignment - Overall Metric",
2234
- "column": "OneIG Overall Score",
2235
- },
2236
- {
2237
- "id": "datapoint_elo_anime",
2238
- "name": "Datapoint ELO - Anime Metric",
2239
- "column": "OneIG Anime Elo",
2240
- },
2241
- {
2242
- "id": "datapoint_elo_human",
2243
- "name": "Datapoint ELO - Human Metric",
2244
- "column": "OneIG Human Elo",
2245
- },
2246
- {
2247
- "id": "datapoint_elo_object",
2248
- "name": "Datapoint ELO - Object Metric",
2249
- "column": "OneIG Object Elo",
2250
- },
2251
- {
2252
- "id": "aa_elo",
2253
- "name": "Artificial Analysis ELO Metric",
2254
- "column": "Artificial Analysis Elo",
2255
- },
2256
- {
2257
- "id": "arena_elo",
2258
- "name": "Arena ELO - Overall Metric",
2259
- "column": "Arena Elo",
2260
- },
2261
- {
2262
- "id": "arena_branding",
2263
- "name": "Arena ELO - Branding / Commercial Metric",
2264
- "column": "Arena Branding / Commercial Elo",
2265
- },
2266
- {
2267
- "id": "arena_3d",
2268
- "name": "Arena ELO - 3D Imaging Metric",
2269
- "column": "Arena 3D Imaging Elo",
2270
- },
2271
- {
2272
- "id": "arena_cartoon",
2273
- "name": "Arena ELO - Cartoon / Anime Metric",
2274
- "column": "Arena Cartoon / Anime Elo",
2275
- },
2276
- {
2277
- "id": "arena_photo",
2278
- "name": "Arena ELO - Photorealistic Metric",
2279
- "column": "Arena Photorealistic Elo",
2280
- },
2281
- {
2282
- "id": "arena_art",
2283
- "name": "Arena ELO - Art Metric",
2284
- "column": "Arena Art Elo",
2285
- },
2286
- {
2287
- "id": "arena_portraits",
2288
- "name": "Arena ELO - Portraits Metric",
2289
- "column": "Arena Portraits Elo",
2290
- },
2291
- {
2292
- "id": "arena_text",
2293
- "name": "Arena ELO - Text Rendering Metric",
2294
- "column": "Arena Text Rendering Elo",
2295
- },
2296
  ]
2297
 
2298
 
@@ -2358,7 +2220,7 @@ datasets = [
2358
  },
2359
  {
2360
  "id": "artificial_analysis",
2361
- "name": "Artificial Analysis",
2362
  "data": aa_df,
2363
  "columns": aa_display_columns,
2364
  "metric_ids": aa_metric_ids,
@@ -2367,7 +2229,7 @@ datasets = [
2367
  },
2368
  {
2369
  "id": "arena_ai",
2370
- "name": "Arena AI",
2371
  "data": arena_df,
2372
  "columns": arena_display_columns,
2373
  "metric_ids": arena_metric_ids,
 
1914
  }
1915
 
1916
 
1917
+ def _as_numeric(df, columns):
1918
+ for column in columns:
1919
+ if column in df.columns:
1920
+ df[column] = pd.to_numeric(df[column], errors="coerce")
1921
+ return df
 
 
 
 
 
 
 
1922
 
1923
 
1924
  def load_oneig_dataframe(path):
 
1926
  df = df.rename(
1927
  columns={
1928
  "Owner": "Endpoint Owner",
 
1929
  "Anime Alignment Score": "OneIG (Anime Alignment)",
1930
  "Human Alignment Score": "OneIG (Human Alignment)",
1931
  "Object Alignment Score": "OneIG (Object Alignment)",
 
1934
  "OneIG (General Object) (Alignment Score)": "OneIG (Object Alignment)",
1935
  }
1936
  )
1937
+ df = df.drop(
1938
+ columns=[
1939
+ column
1940
+ for column in ("URL", "Device", "Optimization", "Optimization Details")
1941
+ if column in df.columns
1942
+ ]
1943
+ )
1944
  if "Optimized" in df.columns:
1945
  df["Optimized"] = df["Optimized"].map(
1946
  {True: "Yes", False: "No", "TRUE": "Yes", "FALSE": "No"}
1947
  ).fillna(df["Optimized"])
1948
 
1949
+ return _as_numeric(
1950
+ df,
1951
+ [
1952
+ "Price / Image (USD)",
1953
+ "Median Generation Time (s)",
1954
+ "Min Generation Time (s)",
1955
+ "OneIG (Anime Alignment)",
1956
+ "OneIG (Human Alignment)",
1957
+ "OneIG (Object Alignment)",
1958
+ "OneIG Anime Elo",
1959
+ "OneIG Human Elo",
1960
+ "OneIG Object Elo",
1961
+ "P-Judge Overall",
1962
+ "Rapidata Elo",
1963
+ ],
1964
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1965
 
1966
 
1967
  def load_artificial_analysis_dataframe(path):
 
1975
  }
1976
  )
1977
  df["Model"] = df["Model"].astype(str).str.strip()
1978
+ return _as_numeric(
1979
+ df, ["Artificial Analysis Elo", "Price / Image (USD)"]
1980
+ ).reset_index(drop=True)
 
 
 
 
 
 
 
 
1981
 
1982
 
1983
  ARENA_CATEGORY_COLUMNS = {
 
2009
  for column in ["Arena Elo", *ARENA_CATEGORY_COLUMNS.values()]
2010
  if column in df.columns
2011
  ]
2012
+ df = _as_numeric(df, score_columns)
 
 
 
 
 
 
2013
  ordered = ["Model", *score_columns]
2014
  return df[[column for column in ordered if column in df.columns]].reset_index(
2015
  drop=True
 
2026
  df = df[~df["Model"].astype(str).str.startswith("#")].copy()
2027
  df["Model"] = df["Model"].astype(str).str.strip()
2028
 
2029
+ return _as_numeric(
2030
+ df,
2031
+ [
2032
+ "Price / Image (USD)",
2033
+ "Median Generation Time (s)",
2034
+ "Min Generation Time (s)",
2035
+ "P-Judge Overall",
2036
+ "Rapidata Elo",
2037
+ "Datapoint Elo",
2038
+ ],
2039
+ ).reset_index(drop=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2040
 
2041
 
2042
  df = load_oneig_dataframe(oneig_path)
 
2055
  oneig_df["OneIG Overall Score"] = oneig_df[oneig_metric_columns].mean(
2056
  axis=1, skipna=True
2057
  )
 
 
 
2058
 
2059
  oneig_display_columns = [
2060
  col
 
2074
  "Median Generation Time (s)",
2075
  "Min Generation Time (s)",
2076
  "Price / Image (USD)",
 
2077
  ]
2078
  if col in oneig_df.columns
2079
  ]
 
2139
  qwen_samples = load_sample_comparison_data(qwen_combined_dir)
2140
 
2141
  metrics = [
2142
+ {"id": "datapoint_elo", "column": "Datapoint Elo"},
2143
+ {"id": "rapidata_elo", "column": "Rapidata Elo"},
2144
+ {"id": "pjudger", "column": "P-Judge Overall"},
2145
+ {"id": "alignment_overall", "column": "OneIG Overall Score"},
2146
+ {"id": "datapoint_elo_anime", "column": "OneIG Anime Elo"},
2147
+ {"id": "datapoint_elo_human", "column": "OneIG Human Elo"},
2148
+ {"id": "datapoint_elo_object", "column": "OneIG Object Elo"},
2149
+ {"id": "aa_elo", "column": "Artificial Analysis Elo"},
2150
+ {"id": "arena_elo", "column": "Arena Elo"},
2151
+ {"id": "arena_branding", "column": "Arena Branding / Commercial Elo"},
2152
+ {"id": "arena_3d", "column": "Arena 3D Imaging Elo"},
2153
+ {"id": "arena_cartoon", "column": "Arena Cartoon / Anime Elo"},
2154
+ {"id": "arena_photo", "column": "Arena Photorealistic Elo"},
2155
+ {"id": "arena_art", "column": "Arena Art Elo"},
2156
+ {"id": "arena_portraits", "column": "Arena Portraits Elo"},
2157
+ {"id": "arena_text", "column": "Arena Text Rendering Elo"},
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2158
  ]
2159
 
2160
 
 
2220
  },
2221
  {
2222
  "id": "artificial_analysis",
2223
+ "name": "Artificial Analysis Dataset",
2224
  "data": aa_df,
2225
  "columns": aa_display_columns,
2226
  "metric_ids": aa_metric_ids,
 
2229
  },
2230
  {
2231
  "id": "arena_ai",
2232
+ "name": "Arena AI Dataset",
2233
  "data": arena_df,
2234
  "columns": arena_display_columns,
2235
  "metric_ids": arena_metric_ids,
model_display.py CHANGED
@@ -12,28 +12,14 @@ import re
12
  MODEL_DISPLAY_NAMES = {
13
  # FLUX
14
  "flux_2_pro": "FLUX.2 [pro]",
15
- "FLUX.2 [pro]": "FLUX.2 [pro]",
16
  "flux_2_max": "FLUX.2 [max]",
17
- "FLUX.2 [max]": "FLUX.2 [max]",
18
  "flux_2_flex": "FLUX.2 [flex]",
19
- "FLUX.2 [flex]": "FLUX.2 [flex]",
20
  "flux_2_dev": "FLUX.2 [dev]",
21
- "FLUX.2 [dev]": "FLUX.2 [dev]",
22
- "FLUX.2 [dev] Turbo": "FLUX.2 [dev] Turbo",
23
- "FLUX.2 [dev] Flash": "FLUX.2 [dev] Flash",
24
  "flux_1_1_pro": "FLUX1.1 [pro]",
25
- "FLUX1.1 [pro]": "FLUX1.1 [pro]",
26
  "flux_1_1_pro_ultra": "FLUX1.1 [pro] Ultra",
27
- "FLUX1.1 [pro] Ultra": "FLUX1.1 [pro] Ultra",
28
  "flux_dev": "FLUX.1 [dev]",
29
- "FLUX.1 [dev]": "FLUX.1 [dev]",
30
  "flux_schnell": "FLUX.1 [schnell]",
31
- "FLUX.1 [schnell]": "FLUX.1 [schnell]",
32
  "flux_krea": "FLUX.1 Krea [dev]",
33
- "FLUX.1 Krea [dev]": "FLUX.1 Krea [dev]",
34
- "FLUX.1 [pro]": "FLUX.1 [pro]",
35
- "FLUX.1 Kontext [pro]": "FLUX.1 Kontext [pro]",
36
- "FLUX.1 Kontext [max]": "FLUX.1 Kontext [max]",
37
  # GPT Image
38
  "gpt_image_2": "GPT Image 2",
39
  "GPT Image 2 (high)": "GPT Image 2",
@@ -52,41 +38,28 @@ MODEL_DISPLAY_NAMES = {
52
  "Nano Banana (Gemini 2.5 Flash Image)": "Nano Banana",
53
  # Seedream
54
  "seedream_5_0": "Seedream 5.0",
55
- "Seedream 5.0 Pro": "Seedream 5.0 Pro",
56
- "Seedream 5.0 Lite": "Seedream 5.0 Lite",
57
  "seedream_4_5": "Seedream 4.5",
58
- "Seedream 4.5": "Seedream 4.5",
59
  "seedream_4_0": "Seedream 4.0",
60
- "Seedream 4.0": "Seedream 4.0",
61
  "seedream_3": "Seedream 3.0",
62
- "Seedream 3.0": "Seedream 3.0",
63
  # Qwen
64
  "qwen_image": "Qwen Image",
65
- "Qwen Image": "Qwen Image",
66
  "qwen_image_2_0_pro": "Qwen Image 2.0 Pro",
67
  "Qwen Image 2.0 Pro (2026-04-22)": "Qwen Image 2.0 Pro",
68
  "Qwen Image 2.0 (2026-03-03)": "Qwen Image 2.0",
69
  "qwen_image_2512": "Qwen Image 2512",
70
  "Qwen Image Max 2512": "Qwen Image 2512",
71
  "qwen_image_fast": "Qwen Image Fast",
72
- "Qwen Image Plus 2601": "Qwen Image Plus 2601",
73
  # Ideogram
74
  "ideogram_4_0_quality": "Ideogram 4.0 Quality",
75
  "Ideogram 4.0 (Quality)": "Ideogram 4.0 Quality",
76
- "Ideogram 4.0": "Ideogram 4.0",
77
- "Ideogram 4.0 Fast": "Ideogram 4.0 Fast",
78
  "Ideogram 4.0 Fast (Quality)": "Ideogram 4.0 Fast Quality",
79
- "Ideogram 4.0 Instant": "Ideogram 4.0 Instant",
80
- "Ideogram 3.0": "Ideogram 3.0",
81
  # Imagen
82
  "imagen_4_0": "Imagen 4",
83
  "imagen_4": "Imagen 4",
84
  "Imagen 4 Standard": "Imagen 4",
85
  "imagen_4_0_ultra": "Imagen 4 Ultra",
86
  "imagen_4_ultra": "Imagen 4 Ultra",
87
- "Imagen 4 Ultra": "Imagen 4 Ultra",
88
  "imagen_4_fast": "Imagen 4 Fast",
89
- "Imagen 4 Fast": "Imagen 4 Fast",
90
  "Imagen 3 (v002)": "Imagen 3",
91
  # HiDream
92
  "hidream_i1_dev": "HiDream-I1 Dev",
@@ -99,11 +72,9 @@ MODEL_DISPLAY_NAMES = {
99
  "HiDream-O1-Image-Dev": "HiDream-O1 Dev",
100
  # Reve
101
  "reve_2_1": "Reve 2.1",
102
- "Reve 2.1": "Reve 2.1",
103
  "Reve Image (Halfmoon)": "Reve Image",
104
  # P-Image
105
  "p_image": "P-Image",
106
- "P-Image": "P-Image",
107
  "p_image_2_ideogram_very_low_1k": "P-Image-Ideogram Very Low 1K",
108
  "p_image_2_ideogram_very_low_2k": "P-Image-Ideogram Very Low 2K",
109
  "P-Image-Ideogram (Very Low)": "P-Image-Ideogram Very Low",
@@ -118,29 +89,21 @@ MODEL_DISPLAY_NAMES = {
118
  "P-Image-Ideogram (High)": "P-Image-Ideogram High",
119
  # Others overlapping P-Bench
120
  "z_image": "Z-Image",
121
- "Z-Image Turbo": "Z-Image Turbo",
122
- "Z-Image Base": "Z-Image Base",
123
  "glm_image": "GLM-Image",
124
- "GLM-Image": "GLM-Image",
125
  "hunyuanimage_3_0": "HunyuanImage 3.0",
126
  "hunyuan_image_3": "HunyuanImage 3.0",
127
  "HunyuanImage 3.0 (Fal)": "HunyuanImage 3.0",
128
  "HunyuanImage 3.0 Instruct (Fal)": "HunyuanImage 3.0 Instruct",
129
- "HunyuanImage 2.1": "HunyuanImage 2.1",
130
  "wan_2_2_image": "Wan 2.2 Image",
131
- "Wan 2.6 Image": "Wan 2.6 Image",
132
  "Wan2.6 Text to Image": "Wan 2.6 Text to Image",
133
  "kling_v2_1": "Kling v2.1",
134
- "Kling Image 3.0 Omni": "Kling Image 3.0 Omni",
135
  "juggernaut_base_flux": "Juggernaut Base FLUX",
136
  "juggernaut_pro_flux": "Juggernaut Pro FLUX",
137
  "juggernaut_lightning_flux": "Juggernaut Lightning FLUX",
138
  "bria_4_fibo": "Bria FIBO",
139
- "Bria 3.2": "Bria 3.2",
140
  "sdxl": "SDXL 1.0",
141
  "Stable Diffusion XL 1.0": "SDXL 1.0",
142
  "sdxl_lightning": "SDXL Lightning",
143
- "SDXL Lightning": "SDXL Lightning",
144
  # Arena AI (kebab / arena.ai ids)
145
  "gpt-image-2 (medium)": "GPT Image 2",
146
  "gpt-image-1.5-high-fidelity": "GPT Image 1.5",
 
12
  MODEL_DISPLAY_NAMES = {
13
  # FLUX
14
  "flux_2_pro": "FLUX.2 [pro]",
 
15
  "flux_2_max": "FLUX.2 [max]",
 
16
  "flux_2_flex": "FLUX.2 [flex]",
 
17
  "flux_2_dev": "FLUX.2 [dev]",
 
 
 
18
  "flux_1_1_pro": "FLUX1.1 [pro]",
 
19
  "flux_1_1_pro_ultra": "FLUX1.1 [pro] Ultra",
 
20
  "flux_dev": "FLUX.1 [dev]",
 
21
  "flux_schnell": "FLUX.1 [schnell]",
 
22
  "flux_krea": "FLUX.1 Krea [dev]",
 
 
 
 
23
  # GPT Image
24
  "gpt_image_2": "GPT Image 2",
25
  "GPT Image 2 (high)": "GPT Image 2",
 
38
  "Nano Banana (Gemini 2.5 Flash Image)": "Nano Banana",
39
  # Seedream
40
  "seedream_5_0": "Seedream 5.0",
 
 
41
  "seedream_4_5": "Seedream 4.5",
 
42
  "seedream_4_0": "Seedream 4.0",
 
43
  "seedream_3": "Seedream 3.0",
 
44
  # Qwen
45
  "qwen_image": "Qwen Image",
 
46
  "qwen_image_2_0_pro": "Qwen Image 2.0 Pro",
47
  "Qwen Image 2.0 Pro (2026-04-22)": "Qwen Image 2.0 Pro",
48
  "Qwen Image 2.0 (2026-03-03)": "Qwen Image 2.0",
49
  "qwen_image_2512": "Qwen Image 2512",
50
  "Qwen Image Max 2512": "Qwen Image 2512",
51
  "qwen_image_fast": "Qwen Image Fast",
 
52
  # Ideogram
53
  "ideogram_4_0_quality": "Ideogram 4.0 Quality",
54
  "Ideogram 4.0 (Quality)": "Ideogram 4.0 Quality",
 
 
55
  "Ideogram 4.0 Fast (Quality)": "Ideogram 4.0 Fast Quality",
 
 
56
  # Imagen
57
  "imagen_4_0": "Imagen 4",
58
  "imagen_4": "Imagen 4",
59
  "Imagen 4 Standard": "Imagen 4",
60
  "imagen_4_0_ultra": "Imagen 4 Ultra",
61
  "imagen_4_ultra": "Imagen 4 Ultra",
 
62
  "imagen_4_fast": "Imagen 4 Fast",
 
63
  "Imagen 3 (v002)": "Imagen 3",
64
  # HiDream
65
  "hidream_i1_dev": "HiDream-I1 Dev",
 
72
  "HiDream-O1-Image-Dev": "HiDream-O1 Dev",
73
  # Reve
74
  "reve_2_1": "Reve 2.1",
 
75
  "Reve Image (Halfmoon)": "Reve Image",
76
  # P-Image
77
  "p_image": "P-Image",
 
78
  "p_image_2_ideogram_very_low_1k": "P-Image-Ideogram Very Low 1K",
79
  "p_image_2_ideogram_very_low_2k": "P-Image-Ideogram Very Low 2K",
80
  "P-Image-Ideogram (Very Low)": "P-Image-Ideogram Very Low",
 
89
  "P-Image-Ideogram (High)": "P-Image-Ideogram High",
90
  # Others overlapping P-Bench
91
  "z_image": "Z-Image",
 
 
92
  "glm_image": "GLM-Image",
 
93
  "hunyuanimage_3_0": "HunyuanImage 3.0",
94
  "hunyuan_image_3": "HunyuanImage 3.0",
95
  "HunyuanImage 3.0 (Fal)": "HunyuanImage 3.0",
96
  "HunyuanImage 3.0 Instruct (Fal)": "HunyuanImage 3.0 Instruct",
 
97
  "wan_2_2_image": "Wan 2.2 Image",
 
98
  "Wan2.6 Text to Image": "Wan 2.6 Text to Image",
99
  "kling_v2_1": "Kling v2.1",
 
100
  "juggernaut_base_flux": "Juggernaut Base FLUX",
101
  "juggernaut_pro_flux": "Juggernaut Pro FLUX",
102
  "juggernaut_lightning_flux": "Juggernaut Lightning FLUX",
103
  "bria_4_fibo": "Bria FIBO",
 
104
  "sdxl": "SDXL 1.0",
105
  "Stable Diffusion XL 1.0": "SDXL 1.0",
106
  "sdxl_lightning": "SDXL Lightning",
 
107
  # Arena AI (kebab / arena.ai ids)
108
  "gpt-image-2 (medium)": "GPT Image 2",
109
  "gpt-image-1.5-high-fidelity": "GPT Image 1.5",
pruna-mascot.png → pruna-logo.png RENAMED
File without changes
ui.py CHANGED
@@ -9,11 +9,11 @@ import plotly.graph_objects as go
9
 
10
  from model_display import display_model_name
11
 
12
- _MASCOT_PATH = Path(__file__).resolve().parent / "pruna-mascot.png"
13
- _MASCOT_DATA_URI = (
14
  "data:image/png;base64,"
15
- + base64.b64encode(_MASCOT_PATH.read_bytes()).decode("ascii")
16
- if _MASCOT_PATH.exists()
17
  else "https://playground.pruna.ai/logo.svg"
18
  )
19
 
@@ -51,13 +51,17 @@ across P-Bench.
51
 
52
  1. Pick a **dataset** and a **metric**.
53
  2. **Leaderboards**: ranked by that metric. Price and generation time sit in
54
- the same table.
55
  3. **Pareto plots**: mark models that are not beaten on both higher score
56
- and lower price (or time).
57
- 4. **Samples**: the same prompts, side by side.
 
 
58
 
59
  ## How a score is made
60
 
 
 
61
  1. Each endpoint is given the same prompt suite.
62
  2. It generates one image per prompt when the run succeeds. Not every model
63
  has every prompt or every metric.
@@ -65,18 +69,33 @@ across P-Bench.
65
  available, by human preference (Datapoint Elo, Rapidata Elo).
66
  4. Price per image and generation time are joined from the evaluation table.
67
 
 
 
 
 
68
  ## Current datasets
69
 
70
  ### Qwen Image Dataset
71
  100 prompts from the 1,000-prompt Qwen Image Bench set, sampled for coverage
72
  across its fine-grained (L3) categories. Metrics include Datapoint Elo,
73
- Rapidata Elo, and P-Judger.
74
 
75
  ### OneIG Alignment Dataset
76
- Prompt-image **alignment** on anime / stylization, human / portrait, and
77
- general object prompts (100 prompts each). This is the alignment slice of
78
  OneIG, not the full suite. Alignment Overall is the mean of the category
79
- scores that exist for that row.
 
 
 
 
 
 
 
 
 
 
 
80
  """
81
 
82
  ABOUT_DETAILS_CONTENT = """
@@ -93,11 +112,17 @@ ABOUT_DETAILS_CONTENT = """
93
  - **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.
94
  Rapidata rejects prompts over 400 characters, so this Elo is on a subset
95
  of each suite (see Setup). Rapidata is not a dataset.
 
 
 
 
 
96
  - **Generation time**: median and minimum generation time in seconds, as
97
  reported in the evaluation table. This is not a p95, and we do not state
98
- warm vs cold or concurrent load.
99
  - **Price**: USD per image in the evaluation table. We do not state list
100
- price vs amount paid, or whether failed generations are included.
 
101
 
102
  Scores from different datasets or metrics are **not interchangeable**. A high
103
  OneIG alignment score is not the same quantity as a Datapoint Elo. Compare
@@ -108,25 +133,29 @@ models *within* a Dataset | Metric view.
108
  - **Evaluation window:** July–August 2026.
109
  - **Update policy:** numbers come from evaluation snapshots in the tables,
110
  not a live API poll.
111
- - **Prompt counts:** OneIG Alignment uses the first 100 prompts from each of
112
- the three categories (300 total). Qwen Image Dataset uses 100 prompts sampled
113
- from the 1,000-prompt pool for roughly even coverage of its fine-grained
114
- (L3) categories.
115
- - **Generation:** one image per prompt per endpoint when the run exists.
116
- Default resolution is 1024×1024. Exceptions: FLUX 1.1 Pro Ultra at 2K,
117
- FLUX 2 Flex at 1008×1008, and any endpoint labeled 2K. The seed is derived
118
- from the prompt, so every model gets the same seed for the same prompt.
119
- Steps, CFG, prompt rewrite, and safety filters follow each endpoint's
120
- default.
121
- - **Datapoint:** every model pair is compared on every prompt, with 10 votes
122
- per battle.
123
- - **Rapidata:** prompts longer than 400 characters are dropped, leaving 212
124
- OneIG prompts and 85 Qwen Image Dataset prompts. 4 votes per pair; about
125
- 26,000 votes on OneIG and 35,000 on Qwen Image Dataset.
 
 
126
 
127
  ## Limits
128
 
129
  - Empty cells mean that track was not run or not reported for that model.
 
 
130
  - Rapidata Elo is not on the full prompt suite, so it is not directly
131
  comparable to Datapoint Elo even on the same dataset.
132
  - Elo ratings can shift when the comparison pool changes: treat them as
@@ -147,6 +176,7 @@ FOOTER_CONTENT = """
147
  <div class="community-footer-links">
148
  <a rel="nofollow" href="https://x.com/PrunaAI" target="_blank">X</a>
149
  <a rel="nofollow" href="https://www.linkedin.com/company/pruna-ai" target="_blank">LinkedIn</a>
 
150
  <a rel="nofollow" href="https://discord.gg/JFQmtFKCjd" target="_blank">Discord</a>
151
  <a rel="nofollow" href="https://github.com/PrunaAI/pruna" target="_blank">GitHub</a>
152
  <a rel="nofollow" href="https://www.pruna.ai/" target="_blank">pruna.ai</a>
@@ -160,7 +190,7 @@ CITATION_CONTENT = """
160
  title={P-Bench: A Leaderboard for Text-to-Image Models},
161
  author={PrunaAI},
162
  year={2026},
163
- howpublished={\\url{https://huggingface.co/spaces/PrunaAI/InferBench}}
164
  }
165
  ```
166
  """
@@ -172,7 +202,7 @@ def render_header():
172
  <header class="app-header">
173
  <div class="app-header-bar">
174
  <div class="app-header-brand">
175
- <img class="app-header-logo" src="{_MASCOT_DATA_URI}" alt="" />
176
  <h1>P-Bench</h1>
177
  </div>
178
  <button type="button" class="theme-toggle" data-mode="dark" aria-label="Switch to light mode" title="Switch to light mode">
@@ -200,11 +230,12 @@ def _item(items, item_id):
200
  return items[0] if items else None
201
 
202
 
203
- def _dataset_choices(datasets, *, require_samples=False):
204
  return [
205
  (dataset["name"], dataset["id"])
206
  for dataset in datasets
207
- if not require_samples or dataset.get("samples")
 
208
  ]
209
 
210
 
@@ -219,6 +250,20 @@ def _dataset_has_pareto(datasets, dataset_id):
219
  return _PARETO_PRICE_COLUMN in columns or _PARETO_TIME_COLUMN in columns
220
 
221
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
222
  ALL_METRICS_ID = "__all__"
223
 
224
 
@@ -306,29 +351,12 @@ def _metric_columns(datasets, metrics, dataset_id):
306
  ]
307
 
308
 
309
- def _view_title(datasets, metrics, dataset_id, metric_id):
310
- dataset = _item(datasets, dataset_id)
311
- dataset_name = dataset["name"] if dataset else "Dataset"
312
- selected = [
313
- item
314
- for item in _normalize_metric_ids(metric_id)
315
- if item != ALL_METRICS_ID
316
- ]
317
- all_ids = [choice[1] for choice in _metric_choices(datasets, metrics, dataset_id)]
318
- if not selected or set(selected) == set(all_ids):
319
- return dataset_name
320
- names = []
321
- for metric_key in selected:
322
- metric = _item(metrics, metric_key)
323
- if metric:
324
- names.append(_display_label(metric["column"]))
325
- if not names:
326
- return dataset_name
327
- if len(names) == 1:
328
- return f"{dataset_name} | {names[0]}"
329
- return f"{dataset_name} | {', '.join(names)}"
330
-
331
-
332
  _LEADERBOARD_META_COLUMNS = [
333
  "Median Generation Time (s)",
334
  "Min Generation Time (s)",
@@ -338,23 +366,14 @@ _LEADERBOARD_META_COLUMNS = [
338
  ]
339
 
340
 
341
- def _columns_for_metric(dataset, metric_column):
342
  """When metrics are selected, show identity + those scores + time/price."""
343
- metric_columns = (
344
- [metric_column]
345
- if isinstance(metric_column, str)
346
- else [column for column in (metric_column or []) if column]
347
- )
348
  available = list(getattr(dataset.get("data"), "columns", [])) or list(
349
  dataset.get("columns") or []
350
  )
351
- identity = [
352
- column
353
- for column in ["Model", "Platform", "Endpoint Owner", "Optimized"]
354
- if column in available
355
- ]
356
  meta = [column for column in _LEADERBOARD_META_COLUMNS if column in available]
357
- scores = [column for column in metric_columns if column in available]
358
  if scores:
359
  return [*identity, *scores, *meta]
360
  return [column for column in (dataset.get("columns") or available) if column != "URL"]
@@ -364,21 +383,14 @@ def resolve_view(datasets, metrics, dataset_id, metric_id):
364
  dataset = _item(datasets, dataset_id)
365
  if not dataset:
366
  return None
367
- metric_ids = _coerce_metric(datasets, metrics, dataset_id, metric_id)
368
- selected_metrics = []
369
- for metric_key in metric_ids:
370
  metric = _item(metrics, metric_key)
371
- if metric:
372
- selected_metrics.append(metric)
373
- score_columns = [
374
- metric["column"]
375
- for metric in selected_metrics
376
- if metric["column"] in getattr(dataset.get("data"), "columns", [])
377
- ]
378
  if score_columns:
379
  columns = _columns_for_metric(dataset, score_columns)
380
- score_column = score_columns[0]
381
- metric = selected_metrics[0]
382
  else:
383
  columns = [
384
  column
@@ -386,16 +398,10 @@ def resolve_view(datasets, metrics, dataset_id, metric_id):
386
  if column != "URL"
387
  ]
388
  score_columns = _metric_columns(datasets, metrics, dataset_id)
389
- score_column = score_columns[0] if score_columns else None
390
- metric = None
391
  return {
392
- "dataset": dataset,
393
- "metric": metric,
394
- "metric_id": metric_ids,
395
- "title": _view_title(datasets, metrics, dataset["id"], metric_ids),
396
  "data": dataset["data"],
397
  "columns": columns,
398
- "score_column": score_column,
399
  "score_columns": score_columns,
400
  "samples": dataset.get("samples"),
401
  "note": dataset.get("note"),
@@ -474,10 +480,8 @@ def _assign_leaderboard_ranks(data, overall_column):
474
  return ranked
475
 
476
 
477
- def _leaderboard_html(data, columns, score_columns, overall_column):
478
- leaderboard = _leaderboard_dataframe(
479
- data, columns, score_columns, overall_column
480
- )
481
  if leaderboard.empty:
482
  return (
483
  '<div class="ranking-table-scroll">'
@@ -546,65 +550,22 @@ def _filter_leaderboard(data, platform, owner, optimized, models=None):
546
  return filtered
547
 
548
 
549
- def _leaderboard_dataframe(data, columns, score_columns, overall_column): # noqa: ARG001
550
- skip_columns = {"URL"}
551
- preferred_prefix = [
552
- column
553
- for column in ["Model", "Platform", "Endpoint Owner", "Optimized"]
554
- if column in data.columns
555
- ]
556
- preferred_suffix = [
557
- column
558
- for column in [
559
- "Median Generation Time (s)",
560
- "Min Generation Time (s)",
561
- "Price / Image (USD)",
562
- "Evaluation Date (UTC)",
563
- "Date",
564
- ]
565
- if column in data.columns
566
- ]
567
- middle = [
568
- column
569
- for column in columns
570
- if column in data.columns
571
- and column not in skip_columns
572
- and column not in preferred_prefix
573
- and column not in preferred_suffix
574
- ]
575
- if (
576
- overall_column
577
- and overall_column in data.columns
578
- and overall_column not in middle
579
- and overall_column not in preferred_prefix
580
- and overall_column not in preferred_suffix
581
- ):
582
- middle.insert(0, overall_column)
583
-
584
- ordered_columns = []
585
- seen = set()
586
- for column in [*preferred_prefix, *middle, *preferred_suffix]:
587
- if column not in seen:
588
- seen.add(column)
589
- ordered_columns.append(column)
590
  if "Rank" in data.columns:
591
- ordered_columns = ["Rank", *[c for c in ordered_columns if c != "Rank"]]
592
-
593
- leaderboard = data[ordered_columns].copy()
594
-
595
- if overall_column and overall_column in leaderboard.columns:
596
- leaderboard = leaderboard.sort_values(
597
- overall_column, ascending=False, na_position="last"
598
- )
599
- leaderboard = leaderboard.reset_index(drop=True)
600
- if "Rank" not in leaderboard.columns:
601
- leaderboard.insert(0, "Rank", leaderboard.index + 1)
602
- return leaderboard.rename(columns=_display_label)
603
 
604
 
605
  def _display_label(column):
606
  labels = {
607
- "_overall_score": "Overall score",
608
  "OneIG Overall Score": "Overall",
609
  "OneIG (Anime Alignment)": "Anime",
610
  "OneIG (Human Alignment)": "Human",
@@ -944,20 +905,15 @@ def _samples_html(samples, selected_models, num_prompts, seed=0):
944
  return _pareto_unavailable_html(
945
  "Samples aren't available for this dataset."
946
  )
947
- models = list(selected_models or [])
948
- available = samples.get("models") or []
949
- models = [model for model in models if model in samples.get("images", {})]
950
  if not models:
951
- models = available[: min(2, len(available))]
952
  return _build_compare_samples_html(samples, models, num_prompts, seed)
953
 
954
 
955
  def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
956
- selected_models = [
957
- model
958
- for model in (selected_models or [])
959
- if model in samples["images"]
960
- ][:MAX_COMPARE_MODELS]
961
 
962
  if not selected_models:
963
  return (
@@ -1023,39 +979,24 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
1023
  return "\n".join(blocks)
1024
 
1025
 
1026
- def _plain_note(note):
1027
- text = (note or "").strip()
1028
- if text.startswith(">"):
1029
- text = text.lstrip(">").strip()
1030
- return text
1031
-
1032
-
1033
  def _leaderboard_intro_markdown(note):
1034
- extra = _plain_note(note)
1035
  parts = [
1036
  "Models are ranked by the selected metric, with price and generation "
1037
  "time in the same table."
1038
  ]
 
1039
  if extra:
1040
  parts.append(extra)
1041
  return "<p class='view-help'>" + " ".join(parts) + "</p>"
1042
 
1043
 
1044
- def _filter_row(
1045
- datasets,
1046
- metrics,
1047
- default_dataset_id,
1048
- default_metric_id=None,
1049
- *,
1050
- require_samples=False,
1051
- include_metric=True,
1052
- ):
1053
  metric_id = _coerce_metric(
1054
  datasets, metrics, default_dataset_id, default_metric_id
1055
  )
1056
  with gr.Row(elem_classes="view-filters"):
1057
  dataset_dd = gr.Dropdown(
1058
- choices=_dataset_choices(datasets, require_samples=require_samples),
1059
  value=default_dataset_id,
1060
  label="Dataset",
1061
  type="value",
@@ -1063,22 +1004,18 @@ def _filter_row(
1063
  scale=2,
1064
  min_width=160,
1065
  )
1066
- metric_dd = None
1067
- if include_metric:
1068
- metric_dd = gr.Dropdown(
1069
- choices=_metric_dropdown_choices(
1070
- datasets, metrics, default_dataset_id
1071
- ),
1072
- value=_metric_dropdown_value(metric_id),
1073
- label="Metric",
1074
- type="value",
1075
- multiselect=True,
1076
- allow_custom_value=False,
1077
- filterable=True,
1078
- scale=2,
1079
- min_width=180,
1080
- elem_classes="filter-chips",
1081
- )
1082
  models_dd = gr.Dropdown(
1083
  choices=_model_choices(datasets, default_dataset_id),
1084
  value=[],
@@ -1101,14 +1038,16 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1101
  initial = resolve_view(datasets, metrics, default_dataset_id, default_metric_id)
1102
  initial_data = initial["data"]
1103
  initial_columns = initial["columns"]
1104
- initial_score_columns = initial["score_columns"]
1105
  initial_samples = initial.get("samples")
1106
  with gr.Column(elem_classes="workspace-shell"):
1107
  with gr.Column(elem_classes="workspace-filters") as filters_host:
1108
  gr.Markdown(
1109
  "<p class='filter-help'>"
1110
  "These filters apply to Leaderboards, Pareto plots, and Samples. "
1111
- "Search in Models, or leave it empty to include every model."
 
 
 
1112
  "</p>",
1113
  elem_classes="filter-help-host",
1114
  )
@@ -1163,12 +1102,9 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1163
  ranking = gr.HTML(
1164
  _leaderboard_html(
1165
  _assign_leaderboard_ranks(
1166
- initial_data,
1167
- initial_score_columns[0] if initial_score_columns else None,
1168
  ),
1169
  initial_columns,
1170
- initial_score_columns,
1171
- initial_score_columns[0] if initial_score_columns else None,
1172
  ),
1173
  padding=False,
1174
  elem_classes="ranking-table-host",
@@ -1451,21 +1387,16 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1451
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
1452
  data = view["data"]
1453
  if include_leaderboard:
1454
- sort_column = view["score_column"] or (
1455
- view["score_columns"][0] if view["score_columns"] else None
1456
- )
1457
  note = _leaderboard_intro_markdown(view.get("note"))
1458
  ranking_html = _leaderboard_html(
1459
  _filter_leaderboard(
1460
- _assign_leaderboard_ranks(data, sort_column),
1461
  platform_value or [],
1462
  owner_value or [],
1463
  optimized_value or [],
1464
  models=models,
1465
  ),
1466
  view["columns"],
1467
- view["score_columns"],
1468
- sort_column,
1469
  )
1470
  else:
1471
  note = gr.skip()
@@ -1495,7 +1426,8 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1495
  samples_visible,
1496
  )
1497
 
1498
- def on_dataset(
 
1499
  dataset_id,
1500
  metric_id,
1501
  models,
@@ -1508,75 +1440,159 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1508
  ):
1509
  view_state = dict(view_state or {})
1510
  tab = view_state.get("current_tab") or TAB_LEADERBOARDS
1511
- dataset_changed = dataset_id != view_state.get("dataset_id")
1512
- synced = _synced_filters(
1513
- dataset_id,
1514
- metric_id,
1515
- models,
1516
- clear_metric=dataset_changed,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1517
  )
1518
- dataset_id, metric_id, models = synced[:3]
1519
  if (
1520
  not dataset_changed
1521
- and _applied_key(view_state)
1522
- == _selection_key(dataset_id, metric_id, models)
1523
  ):
1524
- return _skip_all(len(dataset_outputs))
 
 
 
 
 
 
 
 
1525
  can_pareto = _dataset_has_pareto(datasets, dataset_id)
1526
  can_samples = _dataset_has_samples(datasets, dataset_id)
1527
- selected_tab = tab
1528
- if tab == TAB_SAMPLES and not can_samples:
1529
- selected_tab = TAB_LEADERBOARDS
1530
- elif tab == TAB_PARETO and not can_pareto:
1531
- selected_tab = TAB_LEADERBOARDS
 
 
 
 
 
 
 
 
 
1532
  flags = _content_flags(selected_tab)
1533
- view = resolve_view(datasets, metrics, dataset_id, metric_id)
1534
- extras = _leaderboard_extras(
1535
- view["data"], platform_value, owner_value, optimized_value
1536
- )
1537
- views = _views(
1538
- dataset_id,
1539
- metric_id,
1540
- models,
1541
- extras[3],
1542
- extras[4],
1543
- extras[5],
1544
- num_prompts,
1545
- seed,
1546
- **flags,
1547
- )
1548
  extras_payload = (
1549
  {
1550
- "platform": extras[3],
1551
- "owner": extras[4],
1552
- "optimized": extras[5],
1553
  }
1554
  if selected_tab == TAB_LEADERBOARDS
1555
  else {}
1556
  )
1557
- new_state = _commit_state(
1558
- view_state,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1559
  dataset_id,
1560
  metric_id,
1561
  models,
1562
- selected_tab,
1563
- flags,
1564
- extras=extras_payload,
 
 
 
1565
  )
 
 
 
1566
  return (
1567
- synced[3],
1568
- synced[4],
 
 
 
1569
  extras[6],
1570
  extras[0],
1571
  extras[1],
1572
  extras[2],
1573
- *views,
1574
- gr.update(interactive=can_pareto),
1575
- gr.update(interactive=can_samples),
1576
- gr.update(selected=selected_tab)
1577
- if selected_tab != tab
1578
  else gr.skip(),
1579
- new_state,
1580
  )
1581
 
1582
  def on_metric(
@@ -1590,19 +1606,8 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1590
  seed,
1591
  view_state,
1592
  ):
1593
- view_state = dict(view_state or {})
1594
- tab = view_state.get("current_tab") or TAB_LEADERBOARDS
1595
- selected_raw = _normalize_metric_ids(metric_id)
1596
- metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1597
- models = list(models or [])
1598
- if (
1599
- ALL_METRICS_ID not in selected_raw
1600
- and _applied_key(view_state)
1601
- == _selection_key(dataset_id, metric_id, models)
1602
- ):
1603
- return _skip_all(len(metric_outputs))
1604
- flags = _content_flags(tab)
1605
- views = _views(
1606
  dataset_id,
1607
  metric_id,
1608
  models,
@@ -1611,39 +1616,11 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1611
  optimized_value,
1612
  num_prompts,
1613
  seed,
1614
- **flags,
1615
- )
1616
- extras_payload = (
1617
- {
1618
- "platform": platform_value or [],
1619
- "owner": owner_value or [],
1620
- "optimized": optimized_value or [],
1621
- }
1622
- if tab == TAB_LEADERBOARDS
1623
- else {}
1624
- )
1625
- new_state = _commit_state(
1626
  view_state,
1627
- dataset_id,
1628
- metric_id,
1629
- models,
1630
- tab,
1631
- flags,
1632
- extras=extras_payload,
1633
- )
1634
- metric_update = (
1635
- gr.update(
1636
- choices=_metric_dropdown_choices(datasets, metrics, dataset_id),
1637
- value=_metric_dropdown_value(metric_id),
1638
- )
1639
- if ALL_METRICS_ID in selected_raw
1640
- else gr.skip()
1641
- )
1642
- return (
1643
- metric_update,
1644
- *views,
1645
- new_state,
1646
  )
 
 
 
1647
 
1648
  def on_models(
1649
  dataset_id,
@@ -1656,18 +1633,8 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1656
  seed,
1657
  view_state,
1658
  ):
1659
- view_state = dict(view_state or {})
1660
- tab = view_state.get("current_tab") or TAB_LEADERBOARDS
1661
- metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1662
- incoming = list(models or [])
1663
- model_values = set(
1664
- _model_choice_values(_model_choices(datasets, dataset_id))
1665
- )
1666
- models = [model for model in incoming if model in model_values]
1667
- if _applied_key(view_state) == _selection_key(dataset_id, metric_id, models):
1668
- return _skip_all(len(models_outputs))
1669
- flags = _content_flags(tab)
1670
- views = _views(
1671
  dataset_id,
1672
  metric_id,
1673
  models,
@@ -1676,30 +1643,11 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1676
  optimized_value,
1677
  num_prompts,
1678
  seed,
1679
- **flags,
1680
- )
1681
- extras_payload = (
1682
- {
1683
- "platform": platform_value or [],
1684
- "owner": owner_value or [],
1685
- "optimized": optimized_value or [],
1686
- }
1687
- if tab == TAB_LEADERBOARDS
1688
- else {}
1689
- )
1690
- new_state = _commit_state(
1691
  view_state,
1692
- dataset_id,
1693
- metric_id,
1694
- models,
1695
- tab,
1696
- flags,
1697
- extras=extras_payload,
1698
- )
1699
- models_update = (
1700
- gr.update(value=models) if models != incoming else gr.skip()
1701
  )
1702
- return (models_update, *views, new_state)
 
 
1703
 
1704
  def on_tab_select(
1705
  tab,
@@ -1715,6 +1663,11 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1715
  ):
1716
  view_state = dict(view_state or {})
1717
  prev_tab = view_state.get("current_tab") or TAB_LEADERBOARDS
 
 
 
 
 
1718
  if prev_tab == TAB_LEADERBOARDS:
1719
  _save_leaderboard_filters(
1720
  view_state,
@@ -1722,9 +1675,10 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1722
  owner_value,
1723
  optimized_value,
1724
  )
1725
- view_state["current_tab"] = tab
1726
  metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1727
  models = list(models or [])
 
1728
  view_state["dataset_id"] = dataset_id
1729
  view_state["metric_id"] = metric_id
1730
  view_state["models"] = models
@@ -1756,11 +1710,23 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1756
  else:
1757
  lb_filters = _skip_all(4)
1758
  stale = dict(view_state.get("stale") or {})
1759
- chrome = (filters_vis, metric_vis, *lb_filters)
 
 
 
 
 
 
 
 
 
 
 
1760
  if tab == TAB_ABOUT or not stale.get(tab, True):
1761
  return (
1762
  *chrome,
1763
  *_skip_all(len(view_outputs)),
 
1764
  view_state,
1765
  )
1766
  flags = _content_flags(tab)
@@ -1777,7 +1743,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1777
  )
1778
  stale[tab] = False
1779
  view_state["stale"] = stale
1780
- return (*chrome, *views, view_state)
1781
 
1782
  def on_leaderboard_filters(
1783
  dataset_id,
@@ -1796,21 +1762,16 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1796
  optimized_value,
1797
  )
1798
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
1799
- sort_column = view["score_column"] or (
1800
- view["score_columns"][0] if view["score_columns"] else None
1801
- )
1802
  return (
1803
  _leaderboard_html(
1804
  _filter_leaderboard(
1805
- _assign_leaderboard_ranks(view["data"], sort_column),
1806
  platform_value or [],
1807
  owner_value or [],
1808
  optimized_value or [],
1809
  models=models,
1810
  ),
1811
  view["columns"],
1812
- view["score_columns"],
1813
- sort_column,
1814
  ),
1815
  view_state,
1816
  )
@@ -1913,6 +1874,7 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1913
  filter_inputs = [dataset_dd, metric_dd, models_dd, *view_inputs]
1914
 
1915
  dataset_outputs = [
 
1916
  metric_dd,
1917
  models_dd,
1918
  lb_controls,
@@ -1958,12 +1920,15 @@ def render_image_workspace(datasets, metrics, default_dataset_id, default_metric
1958
 
1959
  tab_outputs = [
1960
  filters_host,
 
1961
  metric_dd,
 
1962
  lb_controls,
1963
  platform,
1964
  owner,
1965
  optimized,
1966
  *view_outputs,
 
1967
  view_state,
1968
  ]
1969
  for tab, tab_item in (
 
9
 
10
  from model_display import display_model_name
11
 
12
+ _LOGO_PATH = Path(__file__).resolve().parent / "pruna-logo.png"
13
+ _LOGO_DATA_URI = (
14
  "data:image/png;base64,"
15
+ + base64.b64encode(_LOGO_PATH.read_bytes()).decode("ascii")
16
+ if _LOGO_PATH.exists()
17
  else "https://playground.pruna.ai/logo.svg"
18
  )
19
 
 
51
 
52
  1. Pick a **dataset** and a **metric**.
53
  2. **Leaderboards**: ranked by that metric. Price and generation time sit in
54
+ the same table when the source publishes them.
55
  3. **Pareto plots**: mark models that are not beaten on both higher score
56
+ and lower price (or time). Only datasets with price or generation time
57
+ can open this tab (not Arena AI).
58
+ 4. **Samples**: the same prompts, side by side. Only for datasets we
59
+ generated (Qwen Image Dataset and OneIG Alignment Dataset).
60
 
61
  ## How a score is made
62
 
63
+ On **Qwen Image Dataset** and **OneIG Alignment Dataset**:
64
+
65
  1. Each endpoint is given the same prompt suite.
66
  2. It generates one image per prompt when the run succeeds. Not every model
67
  has every prompt or every metric.
 
69
  available, by human preference (Datapoint Elo, Rapidata Elo).
70
  4. Price per image and generation time are joined from the evaluation table.
71
 
72
+ **Artificial Analysis** and **Arena AI** are external leaderboards. We import
73
+ their published Elos (and Artificial Analysis price). We do not run their
74
+ prompt suites, so samples are not shown.
75
+
76
  ## Current datasets
77
 
78
  ### Qwen Image Dataset
79
  100 prompts from the 1,000-prompt Qwen Image Bench set, sampled for coverage
80
  across its fine-grained (L3) categories. Metrics include Datapoint Elo,
81
+ Rapidata Elo, and P-Judger. Samples are available.
82
 
83
  ### OneIG Alignment Dataset
84
+ Prompt-image **alignment** on anime / stylization (100), human / portrait
85
+ (100), and general object prompts (99). This is the alignment slice of
86
  OneIG, not the full suite. Alignment Overall is the mean of the category
87
+ scores that exist for that row. Also includes Datapoint category Elo,
88
+ Rapidata Elo, and P-Judger. Samples are available.
89
+
90
+ ### Artificial Analysis Dataset
91
+ External text-to-image Elo and price per image from Artificial Analysis.
92
+ Their prompt set is private, so samples are not shown. Pareto plots use
93
+ price vs score only.
94
+
95
+ ### Arena AI Dataset
96
+ External text-to-image Elo (overall and category) from Arena AI. Their
97
+ prompt set is private, so samples are not shown. Price and generation time
98
+ are not in this export, so Pareto plots are unavailable.
99
  """
100
 
101
  ABOUT_DETAILS_CONTENT = """
 
112
  - **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.
113
  Rapidata rejects prompts over 400 characters, so this Elo is on a subset
114
  of each suite (see Setup). Rapidata is not a dataset.
115
+ - **Artificial Analysis Elo**: Elo published by Artificial Analysis on their
116
+ own dataset.
117
+ - **Arena Elo**: Elo published by Arena AI on their own dataset, plus
118
+ category Elos (branding, 3D, cartoon/anime, photorealistic, art, portraits,
119
+ text rendering).
120
  - **Generation time**: median and minimum generation time in seconds, as
121
  reported in the evaluation table. This is not a p95, and we do not state
122
+ warm vs cold or concurrent load. Not available for Arena AI.
123
  - **Price**: USD per image in the evaluation table. We do not state list
124
+ price vs amount paid, or whether failed generations are included. Not
125
+ available for Arena AI.
126
 
127
  Scores from different datasets or metrics are **not interchangeable**. A high
128
  OneIG alignment score is not the same quantity as a Datapoint Elo. Compare
 
133
  - **Evaluation window:** July–August 2026.
134
  - **Update policy:** numbers come from evaluation snapshots in the tables,
135
  not a live API poll.
136
+ - **Prompt counts:** OneIG Alignment uses 100 anime, 100 human, and 99 object
137
+ prompts (299 total). Qwen Image Dataset uses 100 prompts sampled from the
138
+ 1,000-prompt pool for roughly even coverage of its fine-grained (L3)
139
+ categories. Artificial Analysis and Arena AI use their own private prompt
140
+ sets.
141
+ - **Generation (Qwen and OneIG):** one image per prompt per endpoint when
142
+ the run exists. Default resolution is 1024×1024. Exceptions: FLUX 1.1 Pro
143
+ Ultra at 2K, FLUX 2 Flex at 1008×1008, and any endpoint labeled 2K. The
144
+ seed is derived from the prompt, so every model gets the same seed for the
145
+ same prompt. Steps, CFG, prompt rewrite, and safety filters follow each
146
+ endpoint's default. This does not describe Artificial Analysis or Arena AI.
147
+ - **Datapoint (Qwen and OneIG):** every model pair is compared on every
148
+ prompt, with 10 votes per battle.
149
+ - **Rapidata (Qwen and OneIG):** prompts longer than 400 characters are
150
+ dropped, leaving 212 OneIG prompts and 85 Qwen Image Dataset prompts. 4
151
+ votes per pair; about 26,000 votes on OneIG and 35,000 on Qwen Image
152
+ Dataset.
153
 
154
  ## Limits
155
 
156
  - Empty cells mean that track was not run or not reported for that model.
157
+ - Artificial Analysis and Arena AI samples, prompts, and (for Arena) price
158
+ or latency are not available to P-Bench.
159
  - Rapidata Elo is not on the full prompt suite, so it is not directly
160
  comparable to Datapoint Elo even on the same dataset.
161
  - Elo ratings can shift when the comparison pool changes: treat them as
 
176
  <div class="community-footer-links">
177
  <a rel="nofollow" href="https://x.com/PrunaAI" target="_blank">X</a>
178
  <a rel="nofollow" href="https://www.linkedin.com/company/pruna-ai" target="_blank">LinkedIn</a>
179
+ <a rel="nofollow" href="https://www.instagram.com/pruna.ai/" target="_blank">Instagram</a>
180
  <a rel="nofollow" href="https://discord.gg/JFQmtFKCjd" target="_blank">Discord</a>
181
  <a rel="nofollow" href="https://github.com/PrunaAI/pruna" target="_blank">GitHub</a>
182
  <a rel="nofollow" href="https://www.pruna.ai/" target="_blank">pruna.ai</a>
 
190
  title={P-Bench: A Leaderboard for Text-to-Image Models},
191
  author={PrunaAI},
192
  year={2026},
193
+ howpublished={\\url{https://huggingface.co/spaces/PrunaAI/P-Bench}}
194
  }
195
  ```
196
  """
 
202
  <header class="app-header">
203
  <div class="app-header-bar">
204
  <div class="app-header-brand">
205
+ <img class="app-header-logo" src="{_LOGO_DATA_URI}" alt="" />
206
  <h1>P-Bench</h1>
207
  </div>
208
  <button type="button" class="theme-toggle" data-mode="dark" aria-label="Switch to light mode" title="Switch to light mode">
 
230
  return items[0] if items else None
231
 
232
 
233
+ def _dataset_choices(datasets, *, require_samples=False, require_pareto=False):
234
  return [
235
  (dataset["name"], dataset["id"])
236
  for dataset in datasets
237
+ if (not require_samples or dataset.get("samples"))
238
+ and (not require_pareto or _dataset_has_pareto(datasets, dataset["id"]))
239
  ]
240
 
241
 
 
250
  return _PARETO_PRICE_COLUMN in columns or _PARETO_TIME_COLUMN in columns
251
 
252
 
253
+ def _dataset_dropdown_update(datasets, tab, dataset_id):
254
+ """Limit the dataset list to what the current tab can show."""
255
+ return gr.update(
256
+ choices=_dataset_choices(
257
+ datasets,
258
+ require_samples=tab == TAB_SAMPLES
259
+ and _dataset_has_samples(datasets, dataset_id),
260
+ require_pareto=tab == TAB_PARETO
261
+ and _dataset_has_pareto(datasets, dataset_id),
262
+ ),
263
+ value=dataset_id,
264
+ )
265
+
266
+
267
  ALL_METRICS_ID = "__all__"
268
 
269
 
 
351
  ]
352
 
353
 
354
+ _LEADERBOARD_IDENTITY_COLUMNS = [
355
+ "Model",
356
+ "Platform",
357
+ "Endpoint Owner",
358
+ "Optimized",
359
+ ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
360
  _LEADERBOARD_META_COLUMNS = [
361
  "Median Generation Time (s)",
362
  "Min Generation Time (s)",
 
366
  ]
367
 
368
 
369
+ def _columns_for_metric(dataset, score_columns):
370
  """When metrics are selected, show identity + those scores + time/price."""
 
 
 
 
 
371
  available = list(getattr(dataset.get("data"), "columns", [])) or list(
372
  dataset.get("columns") or []
373
  )
374
+ identity = [column for column in _LEADERBOARD_IDENTITY_COLUMNS if column in available]
 
 
 
 
375
  meta = [column for column in _LEADERBOARD_META_COLUMNS if column in available]
376
+ scores = [column for column in (score_columns or []) if column in available]
377
  if scores:
378
  return [*identity, *scores, *meta]
379
  return [column for column in (dataset.get("columns") or available) if column != "URL"]
 
383
  dataset = _item(datasets, dataset_id)
384
  if not dataset:
385
  return None
386
+ data_columns = getattr(dataset.get("data"), "columns", [])
387
+ score_columns = []
388
+ for metric_key in _coerce_metric(datasets, metrics, dataset_id, metric_id):
389
  metric = _item(metrics, metric_key)
390
+ if metric and metric["column"] in data_columns:
391
+ score_columns.append(metric["column"])
 
 
 
 
 
392
  if score_columns:
393
  columns = _columns_for_metric(dataset, score_columns)
 
 
394
  else:
395
  columns = [
396
  column
 
398
  if column != "URL"
399
  ]
400
  score_columns = _metric_columns(datasets, metrics, dataset_id)
 
 
401
  return {
 
 
 
 
402
  "data": dataset["data"],
403
  "columns": columns,
404
+ "score_column": score_columns[0] if score_columns else None,
405
  "score_columns": score_columns,
406
  "samples": dataset.get("samples"),
407
  "note": dataset.get("note"),
 
480
  return ranked
481
 
482
 
483
+ def _leaderboard_html(data, columns):
484
+ leaderboard = _leaderboard_dataframe(data, columns)
 
 
485
  if leaderboard.empty:
486
  return (
487
  '<div class="ranking-table-scroll">'
 
550
  return filtered
551
 
552
 
553
+ def _leaderboard_dataframe(data, columns):
554
+ ordered = []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
555
  if "Rank" in data.columns:
556
+ ordered.append("Rank")
557
+ for column in columns:
558
+ if (
559
+ column in data.columns
560
+ and column not in {"URL", "Rank"}
561
+ and column not in ordered
562
+ ):
563
+ ordered.append(column)
564
+ return data[ordered].rename(columns=_display_label)
 
 
 
565
 
566
 
567
  def _display_label(column):
568
  labels = {
 
569
  "OneIG Overall Score": "Overall",
570
  "OneIG (Anime Alignment)": "Anime",
571
  "OneIG (Human Alignment)": "Human",
 
905
  return _pareto_unavailable_html(
906
  "Samples aren't available for this dataset."
907
  )
908
+ images = samples.get("images", {})
909
+ models = [model for model in (selected_models or []) if model in images]
 
910
  if not models:
911
+ models = (samples.get("models") or [])[:2]
912
  return _build_compare_samples_html(samples, models, num_prompts, seed)
913
 
914
 
915
  def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
916
+ selected_models = list(selected_models or [])[:MAX_COMPARE_MODELS]
 
 
 
 
917
 
918
  if not selected_models:
919
  return (
 
979
  return "\n".join(blocks)
980
 
981
 
 
 
 
 
 
 
 
982
  def _leaderboard_intro_markdown(note):
 
983
  parts = [
984
  "Models are ranked by the selected metric, with price and generation "
985
  "time in the same table."
986
  ]
987
+ extra = (note or "").strip()
988
  if extra:
989
  parts.append(extra)
990
  return "<p class='view-help'>" + " ".join(parts) + "</p>"
991
 
992
 
993
+ def _filter_row(datasets, metrics, default_dataset_id, default_metric_id=None):
 
 
 
 
 
 
 
 
994
  metric_id = _coerce_metric(
995
  datasets, metrics, default_dataset_id, default_metric_id
996
  )
997
  with gr.Row(elem_classes="view-filters"):
998
  dataset_dd = gr.Dropdown(
999
+ choices=_dataset_choices(datasets),
1000
  value=default_dataset_id,
1001
  label="Dataset",
1002
  type="value",
 
1004
  scale=2,
1005
  min_width=160,
1006
  )
1007
+ metric_dd = gr.Dropdown(
1008
+ choices=_metric_dropdown_choices(datasets, metrics, default_dataset_id),
1009
+ value=_metric_dropdown_value(metric_id),
1010
+ label="Metric",
1011
+ type="value",
1012
+ multiselect=True,
1013
+ allow_custom_value=False,
1014
+ filterable=True,
1015
+ scale=2,
1016
+ min_width=180,
1017
+ elem_classes="filter-chips",
1018
+ )
 
 
 
 
1019
  models_dd = gr.Dropdown(
1020
  choices=_model_choices(datasets, default_dataset_id),
1021
  value=[],
 
1038
  initial = resolve_view(datasets, metrics, default_dataset_id, default_metric_id)
1039
  initial_data = initial["data"]
1040
  initial_columns = initial["columns"]
 
1041
  initial_samples = initial.get("samples")
1042
  with gr.Column(elem_classes="workspace-shell"):
1043
  with gr.Column(elem_classes="workspace-filters") as filters_host:
1044
  gr.Markdown(
1045
  "<p class='filter-help'>"
1046
  "These filters apply to Leaderboards, Pareto plots, and Samples. "
1047
+ "On Samples, only datasets we have generations for are listed. "
1048
+ "On Pareto plots, only datasets with price or generation time "
1049
+ "are listed. Search in Models, or leave it empty to include "
1050
+ "every model."
1051
  "</p>",
1052
  elem_classes="filter-help-host",
1053
  )
 
1102
  ranking = gr.HTML(
1103
  _leaderboard_html(
1104
  _assign_leaderboard_ranks(
1105
+ initial_data, initial.get("score_column")
 
1106
  ),
1107
  initial_columns,
 
 
1108
  ),
1109
  padding=False,
1110
  elem_classes="ranking-table-host",
 
1387
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
1388
  data = view["data"]
1389
  if include_leaderboard:
 
 
 
1390
  note = _leaderboard_intro_markdown(view.get("note"))
1391
  ranking_html = _leaderboard_html(
1392
  _filter_leaderboard(
1393
+ _assign_leaderboard_ranks(data, view["score_column"]),
1394
  platform_value or [],
1395
  owner_value or [],
1396
  optimized_value or [],
1397
  models=models,
1398
  ),
1399
  view["columns"],
 
 
1400
  )
1401
  else:
1402
  note = gr.skip()
 
1426
  samples_visible,
1427
  )
1428
 
1429
+ def _apply_filter_change(
1430
+ source,
1431
  dataset_id,
1432
  metric_id,
1433
  models,
 
1440
  ):
1441
  view_state = dict(view_state or {})
1442
  tab = view_state.get("current_tab") or TAB_LEADERBOARDS
1443
+ selected_raw = _normalize_metric_ids(metric_id)
1444
+ incoming_models = list(models or [])
1445
+ dataset_changed = source == "dataset" and dataset_id != view_state.get(
1446
+ "dataset_id"
1447
+ )
1448
+
1449
+ if source == "dataset":
1450
+ synced = _synced_filters(
1451
+ dataset_id, metric_id, models, clear_metric=dataset_changed
1452
+ )
1453
+ dataset_id, metric_id, models = synced[:3]
1454
+ metric_update, models_update = synced[3], synced[4]
1455
+ else:
1456
+ metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1457
+ if source == "models":
1458
+ allowed = set(
1459
+ _model_choice_values(_model_choices(datasets, dataset_id))
1460
+ )
1461
+ models = [model for model in incoming_models if model in allowed]
1462
+ models_update = (
1463
+ gr.update(value=models)
1464
+ if models != incoming_models
1465
+ else gr.skip()
1466
+ )
1467
+ else:
1468
+ models = incoming_models
1469
+ models_update = gr.skip()
1470
+ metric_update = (
1471
+ gr.update(
1472
+ choices=_metric_dropdown_choices(datasets, metrics, dataset_id),
1473
+ value=_metric_dropdown_value(metric_id),
1474
+ )
1475
+ if source == "metric" and ALL_METRICS_ID in selected_raw
1476
+ else gr.skip()
1477
+ )
1478
+
1479
+ unchanged = _applied_key(view_state) == _selection_key(
1480
+ dataset_id, metric_id, models
1481
  )
 
1482
  if (
1483
  not dataset_changed
1484
+ and not (source == "metric" and ALL_METRICS_ID in selected_raw)
1485
+ and unchanged
1486
  ):
1487
+ return None
1488
+
1489
+ selected_tab = tab
1490
+ extras = (
1491
+ list(platform_value or []),
1492
+ list(owner_value or []),
1493
+ list(optimized_value or []),
1494
+ )
1495
+ extra_updates = None
1496
  can_pareto = _dataset_has_pareto(datasets, dataset_id)
1497
  can_samples = _dataset_has_samples(datasets, dataset_id)
1498
+ if source == "dataset":
1499
+ if tab == TAB_SAMPLES and not can_samples:
1500
+ selected_tab = TAB_LEADERBOARDS
1501
+ elif tab == TAB_PARETO and not can_pareto:
1502
+ selected_tab = TAB_LEADERBOARDS
1503
+ view = resolve_view(datasets, metrics, dataset_id, metric_id)
1504
+ extra_updates = _leaderboard_extras(
1505
+ view["data"] if view else None,
1506
+ platform_value,
1507
+ owner_value,
1508
+ optimized_value,
1509
+ )
1510
+ extras = extra_updates[3:6]
1511
+
1512
  flags = _content_flags(selected_tab)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1513
  extras_payload = (
1514
  {
1515
+ "platform": extras[0],
1516
+ "owner": extras[1],
1517
+ "optimized": extras[2],
1518
  }
1519
  if selected_tab == TAB_LEADERBOARDS
1520
  else {}
1521
  )
1522
+ return {
1523
+ "dataset_id": dataset_id,
1524
+ "metric_update": metric_update,
1525
+ "models_update": models_update,
1526
+ "selected_tab": selected_tab,
1527
+ "tab": tab,
1528
+ "extra_updates": extra_updates,
1529
+ "can_pareto": can_pareto,
1530
+ "can_samples": can_samples,
1531
+ "views": _views(
1532
+ dataset_id,
1533
+ metric_id,
1534
+ models,
1535
+ extras[0],
1536
+ extras[1],
1537
+ extras[2],
1538
+ num_prompts,
1539
+ seed,
1540
+ **flags,
1541
+ ),
1542
+ "state": _commit_state(
1543
+ view_state,
1544
+ dataset_id,
1545
+ metric_id,
1546
+ models,
1547
+ selected_tab,
1548
+ flags,
1549
+ extras=extras_payload,
1550
+ ),
1551
+ }
1552
+
1553
+ def on_dataset(
1554
+ dataset_id,
1555
+ metric_id,
1556
+ models,
1557
+ platform_value,
1558
+ owner_value,
1559
+ optimized_value,
1560
+ num_prompts,
1561
+ seed,
1562
+ view_state,
1563
+ ):
1564
+ result = _apply_filter_change(
1565
+ "dataset",
1566
  dataset_id,
1567
  metric_id,
1568
  models,
1569
+ platform_value,
1570
+ owner_value,
1571
+ optimized_value,
1572
+ num_prompts,
1573
+ seed,
1574
+ view_state,
1575
  )
1576
+ if result is None:
1577
+ return _skip_all(len(dataset_outputs))
1578
+ extras = result["extra_updates"]
1579
  return (
1580
+ _dataset_dropdown_update(
1581
+ datasets, result["selected_tab"], result["dataset_id"]
1582
+ ),
1583
+ result["metric_update"],
1584
+ result["models_update"],
1585
  extras[6],
1586
  extras[0],
1587
  extras[1],
1588
  extras[2],
1589
+ *result["views"],
1590
+ gr.update(interactive=result["can_pareto"]),
1591
+ gr.update(interactive=result["can_samples"]),
1592
+ gr.update(selected=result["selected_tab"])
1593
+ if result["selected_tab"] != result["tab"]
1594
  else gr.skip(),
1595
+ result["state"],
1596
  )
1597
 
1598
  def on_metric(
 
1606
  seed,
1607
  view_state,
1608
  ):
1609
+ result = _apply_filter_change(
1610
+ "metric",
 
 
 
 
 
 
 
 
 
 
 
1611
  dataset_id,
1612
  metric_id,
1613
  models,
 
1616
  optimized_value,
1617
  num_prompts,
1618
  seed,
 
 
 
 
 
 
 
 
 
 
 
 
1619
  view_state,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1620
  )
1621
+ if result is None:
1622
+ return _skip_all(len(metric_outputs))
1623
+ return (result["metric_update"], *result["views"], result["state"])
1624
 
1625
  def on_models(
1626
  dataset_id,
 
1633
  seed,
1634
  view_state,
1635
  ):
1636
+ result = _apply_filter_change(
1637
+ "models",
 
 
 
 
 
 
 
 
 
 
1638
  dataset_id,
1639
  metric_id,
1640
  models,
 
1643
  optimized_value,
1644
  num_prompts,
1645
  seed,
 
 
 
 
 
 
 
 
 
 
 
 
1646
  view_state,
 
 
 
 
 
 
 
 
 
1647
  )
1648
+ if result is None:
1649
+ return _skip_all(len(models_outputs))
1650
+ return (result["models_update"], *result["views"], result["state"])
1651
 
1652
  def on_tab_select(
1653
  tab,
 
1663
  ):
1664
  view_state = dict(view_state or {})
1665
  prev_tab = view_state.get("current_tab") or TAB_LEADERBOARDS
1666
+ requested_tab = tab
1667
+ if tab == TAB_SAMPLES and not _dataset_has_samples(datasets, dataset_id):
1668
+ tab = TAB_LEADERBOARDS
1669
+ elif tab == TAB_PARETO and not _dataset_has_pareto(datasets, dataset_id):
1670
+ tab = TAB_LEADERBOARDS
1671
  if prev_tab == TAB_LEADERBOARDS:
1672
  _save_leaderboard_filters(
1673
  view_state,
 
1675
  owner_value,
1676
  optimized_value,
1677
  )
1678
+ dataset_update = _dataset_dropdown_update(datasets, tab, dataset_id)
1679
  metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
1680
  models = list(models or [])
1681
+ view_state["current_tab"] = tab
1682
  view_state["dataset_id"] = dataset_id
1683
  view_state["metric_id"] = metric_id
1684
  view_state["models"] = models
 
1710
  else:
1711
  lb_filters = _skip_all(4)
1712
  stale = dict(view_state.get("stale") or {})
1713
+ chrome = (
1714
+ filters_vis,
1715
+ dataset_update,
1716
+ metric_vis,
1717
+ gr.skip(),
1718
+ *lb_filters,
1719
+ )
1720
+ tab_select = (
1721
+ gr.update(selected=tab)
1722
+ if tab != requested_tab
1723
+ else gr.skip()
1724
+ )
1725
  if tab == TAB_ABOUT or not stale.get(tab, True):
1726
  return (
1727
  *chrome,
1728
  *_skip_all(len(view_outputs)),
1729
+ tab_select,
1730
  view_state,
1731
  )
1732
  flags = _content_flags(tab)
 
1743
  )
1744
  stale[tab] = False
1745
  view_state["stale"] = stale
1746
+ return (*chrome, *views, tab_select, view_state)
1747
 
1748
  def on_leaderboard_filters(
1749
  dataset_id,
 
1762
  optimized_value,
1763
  )
1764
  view = resolve_view(datasets, metrics, dataset_id, metric_id)
 
 
 
1765
  return (
1766
  _leaderboard_html(
1767
  _filter_leaderboard(
1768
+ _assign_leaderboard_ranks(view["data"], view["score_column"]),
1769
  platform_value or [],
1770
  owner_value or [],
1771
  optimized_value or [],
1772
  models=models,
1773
  ),
1774
  view["columns"],
 
 
1775
  ),
1776
  view_state,
1777
  )
 
1874
  filter_inputs = [dataset_dd, metric_dd, models_dd, *view_inputs]
1875
 
1876
  dataset_outputs = [
1877
+ dataset_dd,
1878
  metric_dd,
1879
  models_dd,
1880
  lb_controls,
 
1920
 
1921
  tab_outputs = [
1922
  filters_host,
1923
+ dataset_dd,
1924
  metric_dd,
1925
+ models_dd,
1926
  lb_controls,
1927
  platform,
1928
  owner,
1929
  optimized,
1930
  *view_outputs,
1931
+ main_tabs,
1932
  view_state,
1933
  ]
1934
  for tab, tab_item in (