Spaces:
Running
ui-updates (#1)
Browse files- style fixes (d5590ebceb9361133bf317e15d76897c1091d28e)
- cleaning css (baa43b90710449175cd743f22621f241258c111f)
- light and dark mode (1bf1b48daa446fc0f2498117fc66bf9d5844635a)
- updating about page (517e6683cfecc16172b4a95611d77b4277d12254)
- refactor: update about tab with experimentation details (3473c50e525da355506889d610dda53eaca89e34)
- fixing mobile rendering (754a2279b6b26d1708a4d33fd8dfbfce08c54825)
- changing inferbench to p-bench (0fa188200802697b6c27337446101c5d73142efe)
- new ui (4b9f11d4ff35dd618331fb51b31778ed7a75bcf8)
- feat: add AA and Arenaai results, fix some ui features (4527ab9a70703593fd674dca571ccbe2c9900633)
- ui updates (3f9b0a536cd48fdd6062945abb5e008f6101b144)
- fixing samples (335823ace0a429bbb7942bb7870a32c80c4bd959)
- cleaning and updating about (a9c18ffb9ca574375a1e1431b8a3c9a911c0aff2)
Co-authored-by: Begüm Çığ <begumcig@users.noreply.huggingface.co>
- README.md +17 -56
- app.py +2118 -542
- data/arena_ai_text_to_image_leaderboard.csv +77 -0
- data/artificial_analysis_text_to_image_leaderboard.csv +146 -0
- data/text_to_image.jsonl +0 -31
- model_display.py +214 -0
- pruna-logo.png +0 -0
- requirements.txt +1 -2
- ui.py +1680 -812
|
@@ -1,5 +1,5 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
emoji: 🥇
|
| 4 |
colorFrom: green
|
| 5 |
colorTo: indigo
|
|
@@ -12,73 +12,34 @@ tags:
|
|
| 12 |
- leaderboard
|
| 13 |
---
|
| 14 |
|
| 15 |
-
#
|
| 16 |
|
| 17 |
-
|
|
|
|
| 18 |
|
| 19 |
-
|
| 20 |
|
| 21 |
-
##
|
| 22 |
|
| 23 |
-
|
| 24 |
-
```
|
| 25 |
-
conda env create -f environment.yml
|
| 26 |
-
```
|
| 27 |
-
|
| 28 |
-
### Install uv
|
| 29 |
-
|
| 30 |
-
Install uv with pip like that:
|
| 31 |
-
|
| 32 |
-
```
|
| 33 |
-
uv venv --python 3.12
|
| 34 |
-
```
|
| 35 |
-
|
| 36 |
-
Then activate the environment:
|
| 37 |
|
| 38 |
```
|
|
|
|
| 39 |
source .venv/bin/activate
|
|
|
|
|
|
|
| 40 |
```
|
| 41 |
|
| 42 |
-
|
| 43 |
|
| 44 |
-
``
|
| 45 |
-
|
| 46 |
-
```
|
| 47 |
|
| 48 |
-
##
|
| 49 |
|
| 50 |
-
|
|
|
|
| 51 |
|
| 52 |
-
This is how you can generate the images.
|
| 53 |
-
```
|
| 54 |
-
python sample.py replicate draw_bench genai_bench geneval hps parti
|
| 55 |
-
```
|
| 56 |
-
|
| 57 |
-
This is how you would evaluate the benchmarks once you have all images:
|
| 58 |
-
```
|
| 59 |
-
python evaluate.py replicate draw_bench genai_bench geneval hps parti
|
| 60 |
```
|
| 61 |
-
|
| 62 |
-
## Leaderboard
|
| 63 |
-
|
| 64 |
-
The leaderboard is [hosted on Hugging Face](https://huggingface.co/spaces/PrunaAI/InferBench/tree/main) with gradio.
|
| 65 |
-
|
| 66 |
-
To run the dashboard locally, you can use the following command:
|
| 67 |
-
|
| 68 |
-
```
|
| 69 |
-
python dashboard/app.py
|
| 70 |
-
```
|
| 71 |
-
|
| 72 |
-
To deploy the dashboard to Hugging Face, you can use the following commands:
|
| 73 |
-
|
| 74 |
-
First, add the remote:
|
| 75 |
-
|
| 76 |
-
```
|
| 77 |
-
git remote add hf https://huggingface.co/spaces/PrunaAI/InferBench
|
| 78 |
-
```
|
| 79 |
-
|
| 80 |
-
Then push the changes of your branch to the remote:
|
| 81 |
-
|
| 82 |
```
|
| 83 |
-
git push hf $(git rev-parse --abbrev-ref HEAD):main --force
|
| 84 |
-
```
|
|
|
|
| 1 |
---
|
| 2 |
+
title: P-Bench
|
| 3 |
emoji: 🥇
|
| 4 |
colorFrom: green
|
| 5 |
colorTo: indigo
|
|
|
|
| 12 |
- leaderboard
|
| 13 |
---
|
| 14 |
|
| 15 |
+
# P-Bench
|
| 16 |
|
| 17 |
+
Compare text-to-image models on quality, speed, and price. This repo is the
|
| 18 |
+
Gradio dashboard: leaderboards, Pareto plots, and side-by-side samples.
|
| 19 |
|
| 20 |
+
The live Space is [PrunaAI/P-Bench](https://huggingface.co/spaces/PrunaAI/P-Bench).
|
| 21 |
|
| 22 |
+
## Run locally
|
| 23 |
|
| 24 |
+
From the repo root:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 25 |
|
| 26 |
```
|
| 27 |
+
python -m venv .venv
|
| 28 |
source .venv/bin/activate
|
| 29 |
+
pip install "gradio==5.19.0" pandas -r requirements.txt
|
| 30 |
+
python app.py
|
| 31 |
```
|
| 32 |
|
| 33 |
+
The app is served at `http://127.0.0.1:7860`.
|
| 34 |
|
| 35 |
+
`requirements.txt` lists Plotly. Gradio and pandas are required locally;
|
| 36 |
+
Hugging Face Spaces installs Gradio from the YAML `sdk_version` above.
|
|
|
|
| 37 |
|
| 38 |
+
## Deploy
|
| 39 |
|
| 40 |
+
`origin` is the Space (`https://huggingface.co/spaces/PrunaAI/P-Bench`).
|
| 41 |
+
Publish the current branch to the live app with:
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
```
|
| 44 |
+
git push origin HEAD:main
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
```
|
|
|
|
|
|
|
@@ -1,429 +1,1470 @@
|
|
|
|
|
| 1 |
import json
|
| 2 |
from collections import defaultdict
|
| 3 |
from pathlib import Path
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
import gradio as gr
|
| 6 |
import pandas as pd
|
| 7 |
|
| 8 |
from ui import (
|
| 9 |
-
render_about,
|
| 10 |
-
render_benchmarks,
|
| 11 |
render_footer,
|
| 12 |
render_header,
|
| 13 |
-
|
| 14 |
)
|
| 15 |
|
|
|
|
| 16 |
custom_css = """
|
| 17 |
-
.
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
width: 100%;
|
| 60 |
-
gap: 16px;
|
| 61 |
-
}
|
| 62 |
-
.benchmark-card {
|
| 63 |
-
width: 100%;
|
| 64 |
-
max-width: none;
|
| 65 |
-
min-height: 220px;
|
| 66 |
-
border: 1px solid rgba(124, 58, 237, 0.45);
|
| 67 |
-
border-left: 4px solid #7c3aed;
|
| 68 |
-
border-radius: 12px;
|
| 69 |
-
background: rgba(139, 92, 246, 0.14);
|
| 70 |
-
padding: 16px 18px;
|
| 71 |
-
box-shadow: 0 4px 12px rgba(76, 29, 149, 0.12);
|
| 72 |
-
}
|
| 73 |
-
.benchmark-card:hover {
|
| 74 |
-
border-color: #5b21b6;
|
| 75 |
-
box-shadow: 0 6px 18px rgba(76, 29, 149, 0.22);
|
| 76 |
-
}
|
| 77 |
-
.community-footer {
|
| 78 |
-
margin-top: 24px;
|
| 79 |
-
padding: 18px 20px;
|
| 80 |
-
border: 1px solid #c4b5fd;
|
| 81 |
-
border-radius: 12px;
|
| 82 |
-
background: rgba(139, 92, 246, 0.16);
|
| 83 |
-
}
|
| 84 |
-
.community-footer h3 {
|
| 85 |
-
margin: 0 0 10px;
|
| 86 |
-
color: #581c87;
|
| 87 |
-
font-size: 1.05rem;
|
| 88 |
}
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
}
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
|
|
|
| 119 |
}
|
| 120 |
-
.
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
font-size: 18px;
|
| 125 |
-
word-break: break-word;
|
| 126 |
}
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
color:
|
| 131 |
-
font-style: normal;
|
| 132 |
-
font-size: 13px;
|
| 133 |
}
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
border-radius: 12px;
|
| 140 |
-
background: #f5f3ff;
|
| 141 |
-
box-shadow: 0 4px 12px rgba(76, 29, 149, 0.12);
|
| 142 |
}
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
|
|
|
| 147 |
}
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 153 |
}
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
}
|
| 161 |
-
.
|
| 162 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 163 |
margin: 8px 0 0;
|
| 164 |
-
padding: 0;
|
|
|
|
| 165 |
}
|
| 166 |
-
.
|
| 167 |
display: grid;
|
| 168 |
-
grid-template-columns:
|
| 169 |
-
gap: 10px;
|
| 170 |
align-items: center;
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
.
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
align-items: center;
|
| 185 |
justify-content: center;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
}
|
| 187 |
-
.
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
color: #4c1d95;
|
| 195 |
-
font-weight: 600;
|
| 196 |
-
}
|
| 197 |
-
.home-empty {
|
| 198 |
-
color: #4c1d95;
|
| 199 |
}
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 204 |
}
|
| 205 |
-
.
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
}
|
| 210 |
-
|
| 211 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
}
|
| 213 |
-
.
|
| 214 |
-
.
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 218 |
}
|
| 219 |
-
.
|
| 220 |
-
.
|
| 221 |
-
.
|
| 222 |
-
.
|
| 223 |
-
|
|
|
|
| 224 |
}
|
| 225 |
-
.
|
| 226 |
-
|
|
|
|
| 227 |
}
|
|
|
|
| 228 |
@media (max-width: 700px) {
|
| 229 |
-
.
|
| 230 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
}
|
| 232 |
}
|
| 233 |
-
|
| 234 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
}
|
| 236 |
-
.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 237 |
width: 100% !important;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
}
|
| 239 |
-
.
|
| 240 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 241 |
width: 100% !important;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 242 |
}
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
|
|
|
|
|
|
| 246 |
}
|
| 247 |
-
.
|
| 248 |
-
|
| 249 |
-
|
| 250 |
-
|
| 251 |
-
|
| 252 |
}
|
| 253 |
-
.
|
| 254 |
-
|
| 255 |
-
border: 1px solid #c4b5fd;
|
| 256 |
-
border-radius: 10px;
|
| 257 |
-
background: rgba(139, 92, 246, 0.14);
|
| 258 |
}
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
|
| 263 |
-
|
| 264 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
}
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 271 |
}
|
|
|
|
|
|
|
| 272 |
.leaderboard-controls {
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
width:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 282 |
}
|
| 283 |
-
.
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
}
|
| 289 |
.ranking-table-scroll {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 290 |
overflow-x: auto;
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
}
|
| 295 |
-
.ranking-table
|
|
|
|
| 296 |
width: 100%;
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 307 |
letter-spacing: 0.04em;
|
| 308 |
text-align: left;
|
| 309 |
text-transform: uppercase;
|
| 310 |
-
}
|
| 311 |
-
.ranking-table th.sortable-col {
|
| 312 |
-
cursor: pointer;
|
| 313 |
-
user-select: none;
|
| 314 |
white-space: nowrap;
|
|
|
|
| 315 |
}
|
| 316 |
-
.ranking-table th.sortable-col
|
| 317 |
-
|
| 318 |
-
}
|
| 319 |
.ranking-table th.sortable-col.sorted-asc::after,
|
| 320 |
.ranking-table th.sortable-col.sorted-desc::after {
|
| 321 |
margin-left: 0.35em;
|
| 322 |
font-size: 0.85em;
|
| 323 |
opacity: 0.85;
|
| 324 |
}
|
| 325 |
-
.ranking-table th.sortable-col.sorted-asc::after {
|
| 326 |
-
|
| 327 |
-
}
|
| 328 |
-
.ranking-table
|
| 329 |
-
|
| 330 |
-
}
|
| 331 |
-
.ranking-table td {
|
| 332 |
-
padding: 13px 14px;
|
| 333 |
-
border-bottom: 1px solid #e9d5ff;
|
| 334 |
-
}
|
| 335 |
-
.ranking-table tbody tr:last-child td {
|
| 336 |
-
border-bottom: 0;
|
| 337 |
-
}
|
| 338 |
-
.ranking-table tbody tr:hover {
|
| 339 |
-
background: #6d28d9;
|
| 340 |
}
|
| 341 |
.ranking-table tbody tr:hover td,
|
| 342 |
.ranking-table tbody tr:hover .rank,
|
| 343 |
.ranking-table tbody tr:hover .metric-score,
|
|
|
|
| 344 |
.ranking-table tbody tr:hover .model-cell strong,
|
| 345 |
-
.ranking-table tbody tr:hover .model-cell span {
|
| 346 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
}
|
| 348 |
-
.ranking-table .rank
|
| 349 |
-
|
| 350 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 351 |
}
|
| 352 |
.ranking-table .model-cell strong,
|
| 353 |
-
.ranking-table .model-cell span {
|
| 354 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 355 |
}
|
| 356 |
.ranking-table .model-cell span {
|
| 357 |
margin-top: 3px;
|
| 358 |
-
color:
|
| 359 |
font-size: 12px;
|
| 360 |
}
|
| 361 |
-
.ranking-table .
|
| 362 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 363 |
font-variant-numeric: tabular-nums;
|
| 364 |
-
font-weight:
|
|
|
|
|
|
|
| 365 |
}
|
|
|
|
| 366 |
.ranking-table .empty-state {
|
| 367 |
-
color:
|
| 368 |
padding: 32px;
|
| 369 |
text-align: center;
|
| 370 |
}
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 392 |
}
|
| 393 |
-
.
|
| 394 |
-
|
| 395 |
-
|
|
|
|
|
|
|
|
|
|
| 396 |
}
|
| 397 |
-
.
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 403 |
}
|
| 404 |
-
.
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
}
|
| 410 |
-
.compare-
|
| 411 |
-
|
| 412 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 413 |
}
|
| 414 |
.compare-prompt-block {
|
| 415 |
margin: 0 0 22px;
|
| 416 |
padding: 14px 16px;
|
| 417 |
-
|
| 418 |
-
border-radius: 12px;
|
| 419 |
-
background: rgba(139, 92, 246, 0.10);
|
| 420 |
}
|
| 421 |
.compare-prompt-meta {
|
| 422 |
display: flex;
|
| 423 |
justify-content: space-between;
|
| 424 |
gap: 12px;
|
| 425 |
margin-bottom: 8px;
|
| 426 |
-
color:
|
| 427 |
font-size: 12px;
|
| 428 |
font-weight: 700;
|
| 429 |
letter-spacing: 0.04em;
|
|
@@ -431,20 +1472,46 @@ custom_css = """
|
|
| 431 |
}
|
| 432 |
.compare-prompt-text {
|
| 433 |
margin: 0 0 14px;
|
| 434 |
-
color:
|
| 435 |
font-size: 15px;
|
| 436 |
line-height: 1.45;
|
| 437 |
}
|
| 438 |
-
.compare-
|
| 439 |
-
|
| 440 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 441 |
}
|
| 442 |
-
.
|
| 443 |
-
|
|
|
|
|
|
|
|
|
|
| 444 |
}
|
|
|
|
| 445 |
.compare-model-label {
|
| 446 |
margin-bottom: 6px;
|
| 447 |
-
color:
|
| 448 |
font-size: 13px;
|
| 449 |
font-weight: 700;
|
| 450 |
word-break: break-word;
|
|
@@ -454,57 +1521,341 @@ custom_css = """
|
|
| 454 |
width: 100%;
|
| 455 |
aspect-ratio: 1 / 1;
|
| 456 |
object-fit: cover;
|
| 457 |
-
border-radius:
|
| 458 |
-
border: 1px solid
|
| 459 |
-
background:
|
| 460 |
}
|
| 461 |
-
.compare-empty
|
| 462 |
-
|
| 463 |
-
|
|
|
|
|
|
|
|
|
|
| 464 |
border-radius: 10px;
|
| 465 |
-
color:
|
| 466 |
-
|
|
|
|
|
|
|
| 467 |
}
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 473 |
}
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 477 |
}
|
| 478 |
-
|
| 479 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 480 |
}
|
| 481 |
-
.
|
| 482 |
-
|
| 483 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 484 |
}
|
| 485 |
-
.
|
| 486 |
-
|
|
|
|
|
|
|
| 487 |
}
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
}
|
| 492 |
}
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 496 |
}
|
| 497 |
"""
|
| 498 |
|
| 499 |
-
theme = gr.
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 506 |
)
|
| 507 |
|
|
|
|
| 508 |
space_root = Path(__file__).resolve().parent
|
| 509 |
data_dir = space_root / "data"
|
| 510 |
|
|
@@ -563,18 +1914,11 @@ def load_sample_comparison_data(folder):
|
|
| 563 |
}
|
| 564 |
|
| 565 |
|
| 566 |
-
def
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
if not links:
|
| 572 |
-
return ""
|
| 573 |
-
|
| 574 |
-
return " ".join(
|
| 575 |
-
f'<a target="_blank" href="{url}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">run {idx}</a>'
|
| 576 |
-
for idx, url in enumerate(links, start=1)
|
| 577 |
-
)
|
| 578 |
|
| 579 |
|
| 580 |
def load_oneig_dataframe(path):
|
|
@@ -582,7 +1926,6 @@ def load_oneig_dataframe(path):
|
|
| 582 |
df = df.rename(
|
| 583 |
columns={
|
| 584 |
"Owner": "Endpoint Owner",
|
| 585 |
-
"Optimization": "Optimization Details",
|
| 586 |
"Anime Alignment Score": "OneIG (Anime Alignment)",
|
| 587 |
"Human Alignment Score": "OneIG (Human Alignment)",
|
| 588 |
"Object Alignment Score": "OneIG (Object Alignment)",
|
|
@@ -591,67 +1934,86 @@ def load_oneig_dataframe(path):
|
|
| 591 |
"OneIG (General Object) (Alignment Score)": "OneIG (Object Alignment)",
|
| 592 |
}
|
| 593 |
)
|
| 594 |
-
|
| 595 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 596 |
if "Optimized" in df.columns:
|
| 597 |
df["Optimized"] = df["Optimized"].map(
|
| 598 |
{True: "Yes", False: "No", "TRUE": "Yes", "FALSE": "No"}
|
| 599 |
).fillna(df["Optimized"])
|
| 600 |
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
|
|
|
| 616 |
|
| 617 |
-
preferred_columns = [
|
| 618 |
-
"Platform",
|
| 619 |
-
"Endpoint Owner",
|
| 620 |
-
"Device",
|
| 621 |
-
"Model",
|
| 622 |
-
"Optimized",
|
| 623 |
-
"Optimization Details",
|
| 624 |
-
"OneIG (Anime Alignment)",
|
| 625 |
-
"OneIG (Human Alignment)",
|
| 626 |
-
"OneIG (Object Alignment)",
|
| 627 |
-
"OneIG Anime Elo",
|
| 628 |
-
"OneIG Human Elo",
|
| 629 |
-
"OneIG Object Elo",
|
| 630 |
-
"Median Inference Time",
|
| 631 |
-
"Median Inference Time (s)",
|
| 632 |
-
"Median Generation Time (s)",
|
| 633 |
-
"Min Generation Time (s)",
|
| 634 |
-
"Median Total Duration (s)",
|
| 635 |
-
"Price per Image",
|
| 636 |
-
"Price / Image (USD)",
|
| 637 |
-
"Evaluation Date (UTC)",
|
| 638 |
-
"URL",
|
| 639 |
-
]
|
| 640 |
-
present_preferred_columns = [col for col in preferred_columns if col in df.columns]
|
| 641 |
-
df = df[
|
| 642 |
-
present_preferred_columns
|
| 643 |
-
+ [col for col in df.columns.tolist() if col not in present_preferred_columns]
|
| 644 |
-
]
|
| 645 |
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 650 |
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 655 |
|
| 656 |
|
| 657 |
def load_qwen_combined_dataframe(path):
|
|
@@ -664,36 +2026,17 @@ def load_qwen_combined_dataframe(path):
|
|
| 664 |
df = df[~df["Model"].astype(str).str.startswith("#")].copy()
|
| 665 |
df["Model"] = df["Model"].astype(str).str.strip()
|
| 666 |
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
| 670 |
-
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
sort_column = next(
|
| 679 |
-
(
|
| 680 |
-
col
|
| 681 |
-
for col in [
|
| 682 |
-
"Datapoint Elo",
|
| 683 |
-
"P-Judge Overall",
|
| 684 |
-
"Rapidata Elo",
|
| 685 |
-
]
|
| 686 |
-
if col in df.columns
|
| 687 |
-
),
|
| 688 |
-
None,
|
| 689 |
-
)
|
| 690 |
-
if sort_column:
|
| 691 |
-
df = df.sort_values(by=sort_column, ascending=False, na_position="last")
|
| 692 |
-
|
| 693 |
-
numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
|
| 694 |
-
for col in numeric_cols:
|
| 695 |
-
df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
|
| 696 |
-
return df.reset_index(drop=True)
|
| 697 |
|
| 698 |
|
| 699 |
df = load_oneig_dataframe(oneig_path)
|
|
@@ -712,9 +2055,6 @@ if oneig_metric_columns:
|
|
| 712 |
oneig_df["OneIG Overall Score"] = oneig_df[oneig_metric_columns].mean(
|
| 713 |
axis=1, skipna=True
|
| 714 |
)
|
| 715 |
-
oneig_df = oneig_df.sort_values(
|
| 716 |
-
by="OneIG Overall Score", ascending=False, na_position="last"
|
| 717 |
-
)
|
| 718 |
|
| 719 |
oneig_display_columns = [
|
| 720 |
col
|
|
@@ -723,6 +2063,7 @@ oneig_display_columns = [
|
|
| 723 |
"Endpoint Owner",
|
| 724 |
"Model",
|
| 725 |
"Optimized",
|
|
|
|
| 726 |
*oneig_metric_columns,
|
| 727 |
"OneIG Anime Elo",
|
| 728 |
"OneIG Human Elo",
|
|
@@ -733,12 +2074,9 @@ oneig_display_columns = [
|
|
| 733 |
"Median Generation Time (s)",
|
| 734 |
"Min Generation Time (s)",
|
| 735 |
"Price / Image (USD)",
|
| 736 |
-
"URL",
|
| 737 |
]
|
| 738 |
if col in oneig_df.columns
|
| 739 |
]
|
| 740 |
-
# Top-level Leaderboard tab uses the same OneIG table.
|
| 741 |
-
display_columns = oneig_display_columns
|
| 742 |
|
| 743 |
oneig_combined_dir = _resolve_data_path(
|
| 744 |
data_dir / "oneig_combined",
|
|
@@ -752,97 +2090,320 @@ qwen_path = _resolve_data_path(
|
|
| 752 |
data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 753 |
space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 754 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 755 |
|
| 756 |
qwen_df = load_qwen_combined_dataframe(qwen_path)
|
| 757 |
-
|
|
|
|
|
|
|
| 758 |
col
|
| 759 |
for col in [
|
| 760 |
-
"
|
| 761 |
"Datapoint Elo",
|
| 762 |
"Rapidata Elo",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 763 |
]
|
| 764 |
if col in qwen_df.columns
|
| 765 |
]
|
| 766 |
-
|
| 767 |
col
|
| 768 |
for col in [
|
| 769 |
"Model",
|
| 770 |
-
|
| 771 |
-
"Raw Win Rate",
|
| 772 |
-
"Median Generation Time (s)",
|
| 773 |
-
"Min Generation Time (s)",
|
| 774 |
"Price / Image (USD)",
|
| 775 |
]
|
| 776 |
-
if col in
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 777 |
]
|
| 778 |
-
qwen_overall_column = (
|
| 779 |
-
"Datapoint Elo"
|
| 780 |
-
if "Datapoint Elo" in qwen_df.columns
|
| 781 |
-
else (qwen_score_columns[0] if qwen_score_columns else None)
|
| 782 |
-
)
|
| 783 |
|
| 784 |
oneig_samples = load_sample_comparison_data(oneig_combined_dir)
|
| 785 |
qwen_samples = load_sample_comparison_data(qwen_combined_dir)
|
| 786 |
|
| 787 |
-
|
| 788 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 789 |
{
|
| 790 |
"id": "oneig",
|
| 791 |
-
"
|
| 792 |
-
"emoji": "🎯",
|
| 793 |
-
"card_description": (
|
| 794 |
-
"Anime/stylization, portrait, and general-object alignment prompts — "
|
| 795 |
-
"alignment scores and Datapoint Elo (not the full OneIG suite), plus "
|
| 796 |
-
"side-by-side generations."
|
| 797 |
-
),
|
| 798 |
-
"intro": (
|
| 799 |
-
"OneIG Alignment covers the alignment slice of OneIG (not every OneIG "
|
| 800 |
-
"dimension). The leaderboard shows category alignment scores and Datapoint "
|
| 801 |
-
"Elo columns; Compare samples uses the combined alignment generations."
|
| 802 |
-
),
|
| 803 |
"data": oneig_df,
|
| 804 |
"columns": oneig_display_columns,
|
| 805 |
-
"
|
| 806 |
-
"overall_column": "OneIG Overall Score",
|
| 807 |
-
"best_score_label": "BEST ONEIG ALIGNMENT",
|
| 808 |
-
"metrics_same_scale": True,
|
| 809 |
"note": (
|
| 810 |
-
"
|
| 811 |
-
"
|
| 812 |
),
|
| 813 |
"samples": oneig_samples,
|
| 814 |
},
|
| 815 |
{
|
| 816 |
-
"id": "
|
| 817 |
-
"
|
| 818 |
-
"
|
| 819 |
-
"
|
| 820 |
-
|
| 821 |
-
|
| 822 |
-
|
| 823 |
-
|
| 824 |
-
|
| 825 |
-
|
| 826 |
-
|
| 827 |
-
|
| 828 |
-
"
|
| 829 |
-
"
|
| 830 |
-
"
|
| 831 |
-
"
|
| 832 |
-
"best_score_label": "BEST DATAPOINT ELO",
|
| 833 |
-
"metrics_same_scale": False,
|
| 834 |
-
"note": (
|
| 835 |
-
"> Models are ordered by Datapoint Elo when available. Other metric "
|
| 836 |
-
"columns come from the P-Judge and Rapidata evaluations on the same "
|
| 837 |
-
"prompt suite."
|
| 838 |
-
),
|
| 839 |
-
"samples": qwen_samples,
|
| 840 |
},
|
| 841 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 842 |
|
| 843 |
|
| 844 |
custom_head = """
|
| 845 |
<script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 846 |
(() => {
|
| 847 |
if (window.__inferbenchLeaderboardSortBound) return;
|
| 848 |
window.__inferbenchLeaderboardSortBound = true;
|
|
@@ -897,25 +2458,40 @@ custom_head = """
|
|
| 897 |
});
|
| 898 |
})();
|
| 899 |
</script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 900 |
"""
|
| 901 |
|
| 902 |
with gr.Blocks(
|
| 903 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 904 |
) as demo:
|
| 905 |
render_header()
|
| 906 |
-
|
| 907 |
-
|
| 908 |
-
|
| 909 |
-
|
| 910 |
-
|
| 911 |
-
|
| 912 |
-
reset_benchmarks,
|
| 913 |
-
outputs=reset_benchmark_outputs,
|
| 914 |
-
)
|
| 915 |
-
with gr.TabItem("About"):
|
| 916 |
-
render_about()
|
| 917 |
render_footer()
|
| 918 |
|
| 919 |
|
| 920 |
if __name__ == "__main__":
|
| 921 |
-
demo.launch(
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
import json
|
| 3 |
from collections import defaultdict
|
| 4 |
from pathlib import Path
|
| 5 |
|
| 6 |
+
# Gradio 5.19's safe_get_lock() returns None on Python 3.12+/3.14 when no
|
| 7 |
+
# event loop exists yet, which breaks /queue/join (pending_message_lock).
|
| 8 |
+
try:
|
| 9 |
+
asyncio.get_running_loop()
|
| 10 |
+
except RuntimeError:
|
| 11 |
+
asyncio.set_event_loop(asyncio.new_event_loop())
|
| 12 |
+
|
| 13 |
import gradio as gr
|
| 14 |
import pandas as pd
|
| 15 |
|
| 16 |
from ui import (
|
|
|
|
|
|
|
| 17 |
render_footer,
|
| 18 |
render_header,
|
| 19 |
+
render_image_workspace,
|
| 20 |
)
|
| 21 |
|
| 22 |
+
# Visual tokens from https://www.pruna.ai/ and https://playground.pruna.ai/
|
| 23 |
custom_css = """
|
| 24 |
+
:root, :host, .dark, :host(.dark) {
|
| 25 |
+
--pruna-bg-primary: #120b1b;
|
| 26 |
+
--pruna-bg-card: #171021;
|
| 27 |
+
--pruna-bg-elevated: #1d1429;
|
| 28 |
+
--pruna-bg-header: #17101f;
|
| 29 |
+
--pruna-border: rgba(74, 57, 98, 0.75);
|
| 30 |
+
--pruna-hairline: rgba(255, 255, 255, 0.08);
|
| 31 |
+
--pruna-text-primary: #fafafa;
|
| 32 |
+
--pruna-text-body: #d4d4d4;
|
| 33 |
+
--pruna-text-muted: #a3a3a3;
|
| 34 |
+
--pruna-accent: #9334e9;
|
| 35 |
+
--pruna-accent-bright: #8f35ff;
|
| 36 |
+
--pruna-lavender: #d8b4fe;
|
| 37 |
+
--pruna-link: #bf8bff;
|
| 38 |
+
--pruna-font: "Inter", ui-sans-serif, system-ui, -apple-system, sans-serif;
|
| 39 |
+
--pruna-glow: radial-gradient(ellipse 60% 35% at 50% -5%, rgba(147, 52, 233, 0.18), transparent 55%);
|
| 40 |
+
--pruna-card-bg: color-mix(in oklab, #1d1429 92%, black 8%);
|
| 41 |
+
--pruna-card-shadow: 0 24px 55px rgba(0, 0, 0, 0.42), inset 0 1px rgba(255, 255, 255, 0.04);
|
| 42 |
+
--pruna-callout-bg: color-mix(in oklab, #171021 82%, black 18%);
|
| 43 |
+
--pruna-menu-border: rgba(255, 255, 255, 0.08);
|
| 44 |
+
--pruna-menu-bg: rgba(255, 255, 255, 0.03);
|
| 45 |
+
--pruna-menu-hover-bg: rgba(255, 255, 255, 0.05);
|
| 46 |
+
--pruna-menu-hover-color: #e4e4e7;
|
| 47 |
+
--pruna-menu-selected-bg: rgba(143, 53, 255, 0.22);
|
| 48 |
+
--pruna-menu-selected-color: #ffffff;
|
| 49 |
+
--pruna-tab-hover: #e4e4e7;
|
| 50 |
+
--pruna-tab-selected: #ffffff;
|
| 51 |
+
--pruna-input-bg: rgba(255, 255, 255, 0.03);
|
| 52 |
+
--pruna-input-border: rgba(255, 255, 255, 0.08);
|
| 53 |
+
--pruna-focus-border: rgba(143, 53, 255, 0.45);
|
| 54 |
+
--pruna-focus-ring: 0 0 0 3px rgba(143, 53, 255, 0.16);
|
| 55 |
+
--pruna-table-hover: rgba(143, 53, 255, 0.16);
|
| 56 |
+
--pruna-table-hover-text: #ffffff;
|
| 57 |
+
--pruna-table-sticky: #17101f;
|
| 58 |
+
--pruna-table-sticky-hover: #2a1844;
|
| 59 |
+
--pruna-rank-bg: rgba(143, 53, 255, 0.22);
|
| 60 |
+
--pruna-rank-ring: 0 0 0 1px rgba(143, 53, 255, 0.45);
|
| 61 |
+
--pruna-score: #ddd6fe;
|
| 62 |
+
--pruna-accordion-bg: rgba(255, 255, 255, 0.02);
|
| 63 |
+
--pruna-accordion-border: rgba(216, 180, 254, 0.15);
|
| 64 |
+
--pruna-dropdown-hover: #2a1844;
|
| 65 |
+
color-scheme: dark;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 66 |
}
|
| 67 |
+
|
| 68 |
+
.light, :host(.light) {
|
| 69 |
+
--pruna-bg-primary: #f6f3fa;
|
| 70 |
+
--pruna-bg-card: #ffffff;
|
| 71 |
+
--pruna-bg-elevated: #ffffff;
|
| 72 |
+
--pruna-bg-header: #f3eef8;
|
| 73 |
+
--pruna-border: rgba(126, 34, 206, 0.16);
|
| 74 |
+
--pruna-hairline: rgba(24, 24, 27, 0.08);
|
| 75 |
+
--pruna-text-primary: #18181b;
|
| 76 |
+
--pruna-text-body: #3f3f46;
|
| 77 |
+
--pruna-text-muted: #71717a;
|
| 78 |
+
--pruna-accent: #9334e9;
|
| 79 |
+
--pruna-accent-bright: #7c22ce;
|
| 80 |
+
--pruna-lavender: #7e22ce;
|
| 81 |
+
--pruna-link: #6d28d9;
|
| 82 |
+
--pruna-glow: radial-gradient(ellipse 60% 35% at 50% -5%, rgba(147, 52, 233, 0.10), transparent 55%);
|
| 83 |
+
--pruna-card-bg: #ffffff;
|
| 84 |
+
--pruna-card-shadow: 0 10px 28px rgba(88, 28, 135, 0.08);
|
| 85 |
+
--pruna-callout-bg: #ffffff;
|
| 86 |
+
--pruna-menu-border: var(--pruna-border);
|
| 87 |
+
--pruna-menu-bg: var(--pruna-bg-card);
|
| 88 |
+
--pruna-menu-hover-bg: rgba(147, 52, 233, 0.06);
|
| 89 |
+
--pruna-menu-hover-color: var(--pruna-text-primary);
|
| 90 |
+
--pruna-menu-selected-bg: rgba(147, 52, 233, 0.12);
|
| 91 |
+
--pruna-menu-selected-color: var(--pruna-accent-bright);
|
| 92 |
+
--pruna-tab-hover: var(--pruna-text-primary);
|
| 93 |
+
--pruna-tab-selected: var(--pruna-text-primary);
|
| 94 |
+
--pruna-input-bg: var(--pruna-bg-card);
|
| 95 |
+
--pruna-input-border: var(--pruna-border);
|
| 96 |
+
--pruna-focus-border: rgba(124, 34, 206, 0.45);
|
| 97 |
+
--pruna-focus-ring: 0 0 0 3px rgba(143, 53, 255, 0.16);
|
| 98 |
+
--pruna-table-hover: rgba(147, 52, 233, 0.08);
|
| 99 |
+
--pruna-table-hover-text: var(--pruna-text-primary);
|
| 100 |
+
--pruna-table-sticky: var(--pruna-bg-card);
|
| 101 |
+
--pruna-table-sticky-hover: #f3e8ff;
|
| 102 |
+
--pruna-rank-bg: rgba(147, 52, 233, 0.12);
|
| 103 |
+
--pruna-rank-ring: 0 0 0 1px rgba(147, 52, 233, 0.28);
|
| 104 |
+
--pruna-score: var(--pruna-accent-bright);
|
| 105 |
+
--pruna-accordion-bg: var(--pruna-bg-card);
|
| 106 |
+
--pruna-accordion-border: var(--pruna-border);
|
| 107 |
+
--pruna-dropdown-hover: #f3e8ff;
|
| 108 |
+
color-scheme: light;
|
| 109 |
}
|
| 110 |
+
|
| 111 |
+
html, body {
|
| 112 |
+
width: 100% !important;
|
| 113 |
+
max-width: 100% !important;
|
| 114 |
+
overflow-x: clip;
|
| 115 |
+
-webkit-tap-highlight-color: transparent;
|
| 116 |
}
|
| 117 |
+
html, body, .gradio-container, .main {
|
| 118 |
+
background: var(--pruna-bg-primary) !important;
|
| 119 |
+
color: var(--pruna-text-primary) !important;
|
| 120 |
+
font-family: var(--pruna-font) !important;
|
|
|
|
|
|
|
| 121 |
}
|
| 122 |
+
button, a, label, input, select, textarea,
|
| 123 |
+
[role="button"], [role="tab"], [role="radio"],
|
| 124 |
+
.gradio-container * {
|
| 125 |
+
-webkit-tap-highlight-color: transparent !important;
|
|
|
|
|
|
|
| 126 |
}
|
| 127 |
+
|
| 128 |
+
.gradio-container {
|
| 129 |
+
--input-radius: 10px;
|
| 130 |
+
--container-radius: 10px;
|
| 131 |
+
--block-radius: 10px;
|
|
|
|
|
|
|
|
|
|
| 132 |
}
|
| 133 |
+
|
| 134 |
+
/* Subtle depth — not a marketing-site hero glow */
|
| 135 |
+
body, .gradio-container {
|
| 136 |
+
background-image: var(--pruna-glow) !important;
|
| 137 |
+
background-attachment: fixed !important;
|
| 138 |
}
|
| 139 |
+
|
| 140 |
+
.gradio-container {
|
| 141 |
+
width: 100% !important;
|
| 142 |
+
max-width: 1200px !important;
|
| 143 |
+
margin: 0 auto !important;
|
| 144 |
+
padding-top: 0 !important;
|
| 145 |
+
padding-left: 20px !important;
|
| 146 |
+
padding-right: 20px !important;
|
| 147 |
+
box-sizing: border-box !important;
|
| 148 |
+
overflow-x: clip;
|
| 149 |
+
}
|
| 150 |
+
.gradio-container .main,
|
| 151 |
+
.gradio-container .wrap,
|
| 152 |
+
.gradio-container .contain,
|
| 153 |
+
.gradio-container .block,
|
| 154 |
+
.gradio-container .form,
|
| 155 |
+
.gradio-container .row,
|
| 156 |
+
.gradio-container .column,
|
| 157 |
+
.gradio-container .gap,
|
| 158 |
+
.gradio-container .html-container,
|
| 159 |
+
.gradio-container .prose,
|
| 160 |
+
.gradio-container .padded,
|
| 161 |
+
.main-tabs,
|
| 162 |
+
.main-tabs .tab-wrapper,
|
| 163 |
+
.main-tabs .tabitem,
|
| 164 |
+
.leaderboard-controls,
|
| 165 |
+
.workspace-shell,
|
| 166 |
+
.workspace-filters,
|
| 167 |
+
.view-filters {
|
| 168 |
+
max-width: 100% !important;
|
| 169 |
}
|
| 170 |
+
|
| 171 |
+
/* —— App header (P-Bench only) —— */
|
| 172 |
+
.app-header-host,
|
| 173 |
+
.app-header-host .html-container,
|
| 174 |
+
.app-header-host .prose,
|
| 175 |
+
.app-header-host.block,
|
| 176 |
+
.app-header-host .padded {
|
| 177 |
+
border: none !important;
|
| 178 |
+
background: transparent !important;
|
| 179 |
+
box-shadow: none !important;
|
| 180 |
+
padding: 0 !important;
|
| 181 |
+
margin: 0 auto !important;
|
| 182 |
+
width: 100% !important;
|
| 183 |
+
max-width: none !important;
|
| 184 |
+
display: block !important;
|
| 185 |
+
text-align: center !important;
|
| 186 |
+
overflow: visible !important;
|
| 187 |
}
|
| 188 |
+
.app-header {
|
| 189 |
+
position: relative;
|
| 190 |
+
display: flex;
|
| 191 |
+
flex-direction: column;
|
| 192 |
+
align-items: stretch;
|
| 193 |
+
width: 100%;
|
| 194 |
margin: 8px 0 0;
|
| 195 |
+
padding: 4px 0 10px;
|
| 196 |
+
text-align: center;
|
| 197 |
}
|
| 198 |
+
.app-header-bar {
|
| 199 |
display: grid;
|
| 200 |
+
grid-template-columns: minmax(36px, 1fr) auto minmax(36px, 1fr);
|
|
|
|
| 201 |
align-items: center;
|
| 202 |
+
width: 100%;
|
| 203 |
+
column-gap: 8px;
|
| 204 |
+
}
|
| 205 |
+
.app-header .theme-toggle,
|
| 206 |
+
button.theme-toggle {
|
| 207 |
+
position: static !important;
|
| 208 |
+
top: auto;
|
| 209 |
+
right: auto;
|
| 210 |
+
grid-column: 3;
|
| 211 |
+
justify-self: end;
|
| 212 |
+
display: inline-flex !important;
|
| 213 |
+
align-items: center !important;
|
| 214 |
+
justify-content: center !important;
|
| 215 |
+
width: 36px !important;
|
| 216 |
+
height: 36px !important;
|
| 217 |
+
min-width: 36px !important;
|
| 218 |
+
min-height: 36px !important;
|
| 219 |
+
margin: 0 !important;
|
| 220 |
+
padding: 0 !important;
|
| 221 |
+
border: 1px solid var(--pruna-menu-border) !important;
|
| 222 |
+
border-radius: 10px !important;
|
| 223 |
+
background: var(--pruna-menu-bg) !important;
|
| 224 |
+
color: var(--pruna-text-muted) !important;
|
| 225 |
+
box-shadow: none !important;
|
| 226 |
+
appearance: none !important;
|
| 227 |
+
-webkit-appearance: none !important;
|
| 228 |
+
box-sizing: border-box !important;
|
| 229 |
+
font: inherit !important;
|
| 230 |
+
line-height: 0 !important;
|
| 231 |
+
cursor: pointer;
|
| 232 |
+
z-index: 6;
|
| 233 |
+
}
|
| 234 |
+
.app-header .theme-toggle:hover,
|
| 235 |
+
.app-header .theme-toggle:active,
|
| 236 |
+
button.theme-toggle:hover,
|
| 237 |
+
button.theme-toggle:active {
|
| 238 |
+
color: var(--pruna-menu-hover-color) !important;
|
| 239 |
+
background: var(--pruna-menu-hover-bg) !important;
|
| 240 |
+
border-color: var(--pruna-menu-border) !important;
|
| 241 |
+
box-shadow: none !important;
|
| 242 |
+
}
|
| 243 |
+
.app-header .theme-toggle:focus,
|
| 244 |
+
button.theme-toggle:focus {
|
| 245 |
+
outline: none !important;
|
| 246 |
+
}
|
| 247 |
+
.app-header .theme-toggle:focus-visible,
|
| 248 |
+
button.theme-toggle:focus-visible {
|
| 249 |
+
color: var(--pruna-menu-hover-color) !important;
|
| 250 |
+
background: var(--pruna-menu-hover-bg) !important;
|
| 251 |
+
border-color: var(--pruna-focus-border) !important;
|
| 252 |
+
box-shadow: var(--pruna-focus-ring) !important;
|
| 253 |
+
}
|
| 254 |
+
.app-header .theme-toggle svg,
|
| 255 |
+
button.theme-toggle svg {
|
| 256 |
+
width: 18px;
|
| 257 |
+
height: 18px;
|
| 258 |
+
flex-shrink: 0;
|
| 259 |
+
}
|
| 260 |
+
.app-header .theme-toggle .theme-icon-sun,
|
| 261 |
+
button.theme-toggle .theme-icon-sun { display: block !important; }
|
| 262 |
+
.app-header .theme-toggle .theme-icon-moon,
|
| 263 |
+
button.theme-toggle .theme-icon-moon { display: none !important; }
|
| 264 |
+
.app-header .theme-toggle[data-mode="light"] .theme-icon-sun,
|
| 265 |
+
button.theme-toggle[data-mode="light"] .theme-icon-sun { display: none !important; }
|
| 266 |
+
.app-header .theme-toggle[data-mode="light"] .theme-icon-moon,
|
| 267 |
+
button.theme-toggle[data-mode="light"] .theme-icon-moon { display: block !important; }
|
| 268 |
+
.app-header-brand {
|
| 269 |
+
display: inline-flex !important;
|
| 270 |
+
flex-direction: row;
|
| 271 |
align-items: center;
|
| 272 |
justify-content: center;
|
| 273 |
+
gap: 12px;
|
| 274 |
+
grid-column: 2;
|
| 275 |
+
justify-self: center;
|
| 276 |
+
width: auto !important;
|
| 277 |
+
max-width: 100%;
|
| 278 |
+
margin: 0;
|
| 279 |
}
|
| 280 |
+
.app-header-logo {
|
| 281 |
+
display: block !important;
|
| 282 |
+
width: 44px !important;
|
| 283 |
+
height: 52px !important;
|
| 284 |
+
margin: 0 !important;
|
| 285 |
+
object-fit: contain;
|
| 286 |
+
flex: 0 0 auto;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
}
|
| 288 |
+
|
| 289 |
+
/* —— Page tabs: centered underline nav —— */
|
| 290 |
+
.workspace-shell,
|
| 291 |
+
.workspace-shell.block,
|
| 292 |
+
.workspace-shell.column,
|
| 293 |
+
.workspace-shell.gap {
|
| 294 |
+
display: flex !important;
|
| 295 |
+
flex-direction: column !important;
|
| 296 |
+
gap: 0 !important;
|
| 297 |
+
padding: 0 !important;
|
| 298 |
+
margin: 0 !important;
|
| 299 |
+
border: none !important;
|
| 300 |
+
background: transparent !important;
|
| 301 |
+
box-shadow: none !important;
|
| 302 |
+
}
|
| 303 |
+
.workspace-shell > .tabs,
|
| 304 |
+
.workspace-shell > .main-tabs,
|
| 305 |
+
.workspace-shell > .block:not(.workspace-filters),
|
| 306 |
+
.workspace-shell .tabs.main-tabs {
|
| 307 |
+
display: contents !important;
|
| 308 |
+
}
|
| 309 |
+
.workspace-filters {
|
| 310 |
+
order: 2 !important;
|
| 311 |
+
}
|
| 312 |
+
.main-tabs .tabitem {
|
| 313 |
+
order: 3 !important;
|
| 314 |
+
}
|
| 315 |
+
.tabs {
|
| 316 |
+
gap: 0 !important;
|
| 317 |
+
}
|
| 318 |
+
.main-tabs > .tab-wrapper {
|
| 319 |
+
order: 1 !important;
|
| 320 |
+
height: auto !important;
|
| 321 |
+
min-height: 0 !important;
|
| 322 |
+
padding: 0 !important;
|
| 323 |
+
margin: 0 0 16px !important;
|
| 324 |
+
justify-content: center !important;
|
| 325 |
+
width: 100% !important;
|
| 326 |
}
|
| 327 |
+
.main-tabs .tab-container {
|
| 328 |
+
height: auto !important;
|
| 329 |
+
justify-content: center !important;
|
| 330 |
+
flex-wrap: wrap !important;
|
| 331 |
+
overflow: hidden !important;
|
| 332 |
+
max-width: 100% !important;
|
| 333 |
+
gap: 2px;
|
| 334 |
+
}
|
| 335 |
+
.main-tabs .tab-container::after {
|
| 336 |
+
background: var(--pruna-hairline) !important;
|
| 337 |
+
}
|
| 338 |
+
.main-tabs .tab-container > button {
|
| 339 |
+
font-family: var(--pruna-font) !important;
|
| 340 |
+
font-size: 0.95rem !important;
|
| 341 |
+
font-weight: 600 !important;
|
| 342 |
+
letter-spacing: -0.015em !important;
|
| 343 |
+
color: var(--pruna-text-muted) !important;
|
| 344 |
+
background: transparent !important;
|
| 345 |
+
border: none !important;
|
| 346 |
+
border-radius: 0 !important;
|
| 347 |
+
box-shadow: none !important;
|
| 348 |
+
padding: 12px 20px !important;
|
| 349 |
+
min-height: 44px !important;
|
| 350 |
+
height: auto !important;
|
| 351 |
+
}
|
| 352 |
+
.main-tabs .tab-container > button:hover:not(.selected),
|
| 353 |
+
.main-tabs .tab-container > button:active:not(.selected) {
|
| 354 |
+
color: var(--pruna-tab-hover) !important;
|
| 355 |
+
background: transparent !important;
|
| 356 |
+
}
|
| 357 |
+
.main-tabs .tab-container > button.selected,
|
| 358 |
+
.main-tabs .tab-container > button[aria-selected="true"] {
|
| 359 |
+
color: var(--pruna-tab-selected) !important;
|
| 360 |
+
background: transparent !important;
|
| 361 |
+
}
|
| 362 |
+
.main-tabs .tab-container > button:disabled,
|
| 363 |
+
.main-tabs .tab-container > button[disabled] {
|
| 364 |
+
opacity: 0.4 !important;
|
| 365 |
+
color: var(--pruna-text-muted) !important;
|
| 366 |
+
cursor: not-allowed !important;
|
| 367 |
+
}
|
| 368 |
+
.main-tabs .tab-container > button.selected::after,
|
| 369 |
+
.main-tabs .tab-container > button[aria-selected="true"]::after {
|
| 370 |
+
background: var(--pruna-accent-bright) !important;
|
| 371 |
+
height: 2px !important;
|
| 372 |
+
}
|
| 373 |
+
.main-tabs .tabitem {
|
| 374 |
+
padding: 8px 0 0 !important;
|
| 375 |
+
border-radius: 0 !important;
|
| 376 |
+
overflow: visible !important;
|
| 377 |
}
|
| 378 |
+
|
| 379 |
+
/* —— About: two columns on desktop, stacked on phone —— */
|
| 380 |
+
.about-layout,
|
| 381 |
+
.about-layout.row,
|
| 382 |
+
.about-layout .form {
|
| 383 |
+
display: flex !important;
|
| 384 |
+
flex-wrap: wrap !important;
|
| 385 |
+
align-items: flex-start !important;
|
| 386 |
+
gap: 8px 40px !important;
|
| 387 |
+
width: 100% !important;
|
| 388 |
}
|
| 389 |
+
.about-layout > .about-col,
|
| 390 |
+
.about-layout .form > .about-col {
|
| 391 |
+
flex: 1 1 340px !important;
|
| 392 |
+
min-width: 280px !important;
|
| 393 |
+
max-width: 100% !important;
|
| 394 |
+
}
|
| 395 |
+
.about-copy,
|
| 396 |
+
.about-copy.block,
|
| 397 |
+
.about-copy .prose,
|
| 398 |
+
.about-copy.prose,
|
| 399 |
+
.about-copy.padded {
|
| 400 |
+
border: none !important;
|
| 401 |
+
background: transparent !important;
|
| 402 |
+
box-shadow: none !important;
|
| 403 |
+
padding: 0 !important;
|
| 404 |
+
margin: 0 !important;
|
| 405 |
+
width: 100% !important;
|
| 406 |
+
max-width: 100% !important;
|
| 407 |
+
overflow-wrap: break-word !important;
|
| 408 |
+
word-break: break-word !important;
|
| 409 |
}
|
| 410 |
+
.about-copy ol,
|
| 411 |
+
.about-copy ul,
|
| 412 |
+
.about-copy .prose ol,
|
| 413 |
+
.about-copy .prose ul {
|
| 414 |
+
padding-left: 1.25rem !important;
|
| 415 |
+
margin: 0 0 0.8rem !important;
|
| 416 |
}
|
| 417 |
+
.about-copy li,
|
| 418 |
+
.about-copy .prose li {
|
| 419 |
+
overflow-wrap: break-word;
|
| 420 |
}
|
| 421 |
+
|
| 422 |
@media (max-width: 700px) {
|
| 423 |
+
.gradio-container {
|
| 424 |
+
padding-left: 12px !important;
|
| 425 |
+
padding-right: 12px !important;
|
| 426 |
+
}
|
| 427 |
+
.gradio-container .block,
|
| 428 |
+
.gradio-container .form,
|
| 429 |
+
.gradio-container .row,
|
| 430 |
+
.gradio-container .column,
|
| 431 |
+
.gradio-container .wrap,
|
| 432 |
+
.leaderboard-controls,
|
| 433 |
+
.view-filters {
|
| 434 |
+
min-width: 0 !important;
|
| 435 |
+
max-width: 100% !important;
|
| 436 |
+
}
|
| 437 |
+
.app-header {
|
| 438 |
+
padding: 4px 0 8px;
|
| 439 |
+
margin: 4px 0 0;
|
| 440 |
+
}
|
| 441 |
+
.app-header-logo {
|
| 442 |
+
width: 36px !important;
|
| 443 |
+
height: 42px !important;
|
| 444 |
+
}
|
| 445 |
+
.app-header-brand h1,
|
| 446 |
+
.gradio-container .app-header-brand h1 {
|
| 447 |
+
font-size: 1.55rem !important;
|
| 448 |
+
}
|
| 449 |
+
.app-header-tagline {
|
| 450 |
+
font-size: 0.88rem !important;
|
| 451 |
+
padding: 0 8px !important;
|
| 452 |
+
}
|
| 453 |
+
.main-tabs > .tab-wrapper {
|
| 454 |
+
margin: 4px 0 14px !important;
|
| 455 |
+
}
|
| 456 |
+
.main-tabs .tab-container {
|
| 457 |
+
flex-wrap: wrap !important;
|
| 458 |
+
justify-content: center !important;
|
| 459 |
+
gap: 0 !important;
|
| 460 |
+
}
|
| 461 |
+
.main-tabs .tab-container > button {
|
| 462 |
+
padding: 10px 12px !important;
|
| 463 |
+
font-size: 0.88rem !important;
|
| 464 |
+
min-height: 40px !important;
|
| 465 |
+
}
|
| 466 |
+
.about-layout,
|
| 467 |
+
.about-layout.row,
|
| 468 |
+
.about-layout .form {
|
| 469 |
+
flex-direction: column !important;
|
| 470 |
+
flex-wrap: nowrap !important;
|
| 471 |
+
gap: 8px !important;
|
| 472 |
+
}
|
| 473 |
+
.about-layout > .about-col,
|
| 474 |
+
.about-layout > div {
|
| 475 |
+
flex: 1 1 auto !important;
|
| 476 |
+
width: 100% !important;
|
| 477 |
+
}
|
| 478 |
+
.about-copy h1,
|
| 479 |
+
.about-copy .prose h1 {
|
| 480 |
+
font-size: 1.3rem !important;
|
| 481 |
+
}
|
| 482 |
+
.about-copy h2,
|
| 483 |
+
.about-copy .prose h2 {
|
| 484 |
+
margin-top: 1.25rem !important;
|
| 485 |
+
font-size: 1.12rem !important;
|
| 486 |
+
}
|
| 487 |
+
.community-footer-links {
|
| 488 |
+
gap: 10px 14px;
|
| 489 |
+
}
|
| 490 |
+
.leaderboard-controls,
|
| 491 |
+
.leaderboard-controls.row,
|
| 492 |
+
.leaderboard-controls .form,
|
| 493 |
+
.view-filters,
|
| 494 |
+
.view-filters.row,
|
| 495 |
+
.view-filters .form {
|
| 496 |
+
flex-direction: column !important;
|
| 497 |
+
align-items: stretch !important;
|
| 498 |
+
}
|
| 499 |
+
.leaderboard-controls > div,
|
| 500 |
+
.leaderboard-controls .form > div,
|
| 501 |
+
.view-filters > div,
|
| 502 |
+
.view-filters .form > div {
|
| 503 |
+
flex: 1 1 auto !important;
|
| 504 |
+
width: 100% !important;
|
| 505 |
+
max-width: 100% !important;
|
| 506 |
+
}
|
| 507 |
+
.ranking-table-scroll {
|
| 508 |
+
max-height: min(75vh, 640px);
|
| 509 |
+
width: 100%;
|
| 510 |
+
}
|
| 511 |
+
.ranking-table th,
|
| 512 |
+
.ranking-table td,
|
| 513 |
+
.prose .ranking-table th,
|
| 514 |
+
.prose .ranking-table td {
|
| 515 |
+
padding: 8px 10px !important;
|
| 516 |
+
}
|
| 517 |
+
.ranking-table .rank,
|
| 518 |
+
.prose .ranking-table .rank,
|
| 519 |
+
.ranking-table th.rank {
|
| 520 |
+
position: sticky !important;
|
| 521 |
+
left: 0 !important;
|
| 522 |
+
width: 2.4rem;
|
| 523 |
+
min-width: 2.4rem;
|
| 524 |
+
}
|
| 525 |
+
.ranking-table .model-cell,
|
| 526 |
+
.prose .ranking-table .model-cell,
|
| 527 |
+
.ranking-table th.model-cell {
|
| 528 |
+
position: sticky !important;
|
| 529 |
+
left: 2.4rem !important;
|
| 530 |
+
min-width: 108px;
|
| 531 |
+
max-width: 36vw;
|
| 532 |
+
}
|
| 533 |
+
.ranking-table .model-cell strong {
|
| 534 |
+
white-space: nowrap;
|
| 535 |
+
}
|
| 536 |
+
.compare-controls,
|
| 537 |
+
.compare-controls.row,
|
| 538 |
+
.compare-controls .form {
|
| 539 |
+
flex-direction: column !important;
|
| 540 |
+
flex-wrap: nowrap !important;
|
| 541 |
+
flex: 0 0 auto !important;
|
| 542 |
+
align-items: stretch !important;
|
| 543 |
+
align-content: flex-start !important;
|
| 544 |
+
height: auto !important;
|
| 545 |
+
min-height: 0 !important;
|
| 546 |
+
gap: 10px !important;
|
| 547 |
+
margin-bottom: 12px !important;
|
| 548 |
+
}
|
| 549 |
+
.compare-controls .compare-models,
|
| 550 |
+
.compare-controls .compare-prompt-count,
|
| 551 |
+
.compare-controls .compare-shuffle,
|
| 552 |
+
.compare-controls .form > div {
|
| 553 |
+
flex: 0 0 auto !important;
|
| 554 |
+
width: 100% !important;
|
| 555 |
+
max-width: 100% !important;
|
| 556 |
+
min-width: 0 !important;
|
| 557 |
+
height: auto !important;
|
| 558 |
+
min-height: 0 !important;
|
| 559 |
+
align-self: stretch !important;
|
| 560 |
+
}
|
| 561 |
+
.compare-controls .compare-prompt-count {
|
| 562 |
+
max-width: 100% !important;
|
| 563 |
+
}
|
| 564 |
+
.compare-controls .compare-models .wrap,
|
| 565 |
+
.compare-controls .compare-models .wrap-inner {
|
| 566 |
+
height: auto !important;
|
| 567 |
+
min-height: 40px !important;
|
| 568 |
+
}
|
| 569 |
+
.compare-controls .compare-shuffle,
|
| 570 |
+
.compare-controls .compare-shuffle button {
|
| 571 |
+
width: 100% !important;
|
| 572 |
+
align-self: stretch !important;
|
| 573 |
+
margin-top: 0 !important;
|
| 574 |
+
}
|
| 575 |
+
.compare-row {
|
| 576 |
+
grid-template-columns: 1fr !important;
|
| 577 |
+
}
|
| 578 |
+
.compare-prompt-meta {
|
| 579 |
+
flex-wrap: wrap;
|
| 580 |
+
}
|
| 581 |
+
.pareto-layout,
|
| 582 |
+
.pareto-layout.row,
|
| 583 |
+
.pareto-layout .form {
|
| 584 |
+
flex-direction: column !important;
|
| 585 |
+
}
|
| 586 |
+
.pareto-layout > .pareto-col,
|
| 587 |
+
.pareto-layout > div,
|
| 588 |
+
.pareto-layout .form > div {
|
| 589 |
+
flex: 1 1 auto !important;
|
| 590 |
+
width: 100% !important;
|
| 591 |
}
|
| 592 |
}
|
| 593 |
+
|
| 594 |
+
/* —— Workspace filters (dataset / metric / models) —— */
|
| 595 |
+
.workspace-filters,
|
| 596 |
+
.workspace-filters.block,
|
| 597 |
+
.workspace-filters.column,
|
| 598 |
+
.workspace-filters.gap {
|
| 599 |
+
margin: 0 0 12px;
|
| 600 |
+
padding: 0 !important;
|
| 601 |
+
gap: 12px !important;
|
| 602 |
+
overflow: visible !important;
|
| 603 |
+
}
|
| 604 |
+
.view-filters {
|
| 605 |
+
display: flex !important;
|
| 606 |
+
flex-wrap: wrap !important;
|
| 607 |
+
align-items: end !important;
|
| 608 |
+
gap: 12px !important;
|
| 609 |
+
margin: 0;
|
| 610 |
+
overflow: visible !important;
|
| 611 |
+
}
|
| 612 |
+
.view-filters > div,
|
| 613 |
+
.view-filters > .block,
|
| 614 |
+
.view-filters > .form {
|
| 615 |
+
flex: 1 1 0 !important;
|
| 616 |
+
min-width: 0 !important;
|
| 617 |
+
}
|
| 618 |
+
.view-filters > .block,
|
| 619 |
+
.view-filters .form,
|
| 620 |
+
.view-filters .block {
|
| 621 |
+
border: none !important;
|
| 622 |
+
background: transparent !important;
|
| 623 |
+
box-shadow: none !important;
|
| 624 |
+
padding: 0 !important;
|
| 625 |
+
overflow: visible !important;
|
| 626 |
+
--block-border-width: 0 !important;
|
| 627 |
+
}
|
| 628 |
+
.view-filters label {
|
| 629 |
+
display: block !important;
|
| 630 |
+
margin: 0 !important;
|
| 631 |
+
padding: 0 0 8px !important;
|
| 632 |
+
color: var(--pruna-text-muted) !important;
|
| 633 |
+
font-size: 0.8rem !important;
|
| 634 |
+
font-weight: 500 !important;
|
| 635 |
+
line-height: 1.2 !important;
|
| 636 |
+
}
|
| 637 |
+
.view-filters .wrap {
|
| 638 |
+
min-width: 0 !important;
|
| 639 |
+
border: none !important;
|
| 640 |
+
background: transparent !important;
|
| 641 |
+
box-shadow: none !important;
|
| 642 |
+
overflow: visible !important;
|
| 643 |
+
}
|
| 644 |
+
.view-filters .wrap-inner {
|
| 645 |
+
display: flex !important;
|
| 646 |
+
flex-wrap: nowrap !important;
|
| 647 |
+
align-items: center !important;
|
| 648 |
+
gap: 6px !important;
|
| 649 |
+
min-height: 40px !important;
|
| 650 |
+
height: 40px !important;
|
| 651 |
+
padding: 0 10px !important;
|
| 652 |
+
border: 1px solid var(--pruna-input-border) !important;
|
| 653 |
+
border-radius: 10px !important;
|
| 654 |
+
background: var(--pruna-input-bg) !important;
|
| 655 |
+
box-shadow: none !important;
|
| 656 |
+
overflow: hidden !important;
|
| 657 |
+
}
|
| 658 |
+
.view-filters .secondary-wrap {
|
| 659 |
+
display: flex !important;
|
| 660 |
+
flex-wrap: nowrap !important;
|
| 661 |
+
align-items: center !important;
|
| 662 |
+
flex: 1 1 auto !important;
|
| 663 |
+
min-width: 0 !important;
|
| 664 |
+
height: 40px !important;
|
| 665 |
+
margin: 0 !important;
|
| 666 |
+
padding: 0 !important;
|
| 667 |
+
border: none !important;
|
| 668 |
+
background: transparent !important;
|
| 669 |
+
box-shadow: none !important;
|
| 670 |
+
overflow: visible !important;
|
| 671 |
+
}
|
| 672 |
+
.view-filters .filter-chips .wrap-inner {
|
| 673 |
+
overflow-x: auto !important;
|
| 674 |
+
overflow-y: hidden !important;
|
| 675 |
+
overscroll-behavior-x: contain;
|
| 676 |
+
scrollbar-width: thin;
|
| 677 |
+
scrollbar-color: var(--pruna-border) transparent;
|
| 678 |
+
}
|
| 679 |
+
.view-filters .filter-chips .secondary-wrap {
|
| 680 |
+
min-width: 2.5rem !important;
|
| 681 |
+
}
|
| 682 |
+
.view-filters .filter-chips .wrap-inner::-webkit-scrollbar {
|
| 683 |
+
height: 6px;
|
| 684 |
+
}
|
| 685 |
+
.view-filters .filter-chips .wrap-inner::-webkit-scrollbar-thumb {
|
| 686 |
+
background: var(--pruna-border);
|
| 687 |
+
border-radius: 99px;
|
| 688 |
+
}
|
| 689 |
+
.view-filters .filter-chips .token {
|
| 690 |
+
flex: 0 0 auto !important;
|
| 691 |
+
white-space: nowrap !important;
|
| 692 |
+
word-break: keep-all !important;
|
| 693 |
+
}
|
| 694 |
+
.view-filters .filter-chips .token span {
|
| 695 |
+
white-space: nowrap !important;
|
| 696 |
+
overflow: hidden !important;
|
| 697 |
+
text-overflow: ellipsis !important;
|
| 698 |
+
max-width: 14rem;
|
| 699 |
+
}
|
| 700 |
+
.view-filters .token:empty {
|
| 701 |
+
display: none !important;
|
| 702 |
}
|
| 703 |
+
.view-filters input {
|
| 704 |
+
border: none !important;
|
| 705 |
+
background: transparent !important;
|
| 706 |
+
box-shadow: none !important;
|
| 707 |
+
min-height: 28px !important;
|
| 708 |
+
height: 28px !important;
|
| 709 |
width: 100% !important;
|
| 710 |
+
min-width: 0 !important;
|
| 711 |
+
padding: 0 4px !important;
|
| 712 |
+
line-height: 28px !important;
|
| 713 |
+
}
|
| 714 |
+
.view-filters .dropdown-arrow,
|
| 715 |
+
.view-filters .icon-wrap {
|
| 716 |
+
flex: 0 0 auto !important;
|
| 717 |
+
align-self: center !important;
|
| 718 |
+
margin-left: auto !important;
|
| 719 |
+
}
|
| 720 |
+
.view-title,
|
| 721 |
+
.view-title.block,
|
| 722 |
+
.view-title .padded {
|
| 723 |
+
border: none !important;
|
| 724 |
+
background: transparent !important;
|
| 725 |
+
box-shadow: none !important;
|
| 726 |
+
padding: 0 !important;
|
| 727 |
+
margin: 0 0 8px !important;
|
| 728 |
+
}
|
| 729 |
+
.view-title h1,
|
| 730 |
+
.view-title .prose h1,
|
| 731 |
+
.gradio-container .view-title h1 {
|
| 732 |
+
margin: 0.35rem 0 0.4rem !important;
|
| 733 |
+
font-size: 1.45rem !important;
|
| 734 |
+
}
|
| 735 |
+
.filter-help-host,
|
| 736 |
+
.filter-help-host.block,
|
| 737 |
+
.filter-help-host .padded,
|
| 738 |
+
.filter-help-host .html-container,
|
| 739 |
+
.filter-help-host .prose {
|
| 740 |
+
border: none !important;
|
| 741 |
+
background: transparent !important;
|
| 742 |
+
box-shadow: none !important;
|
| 743 |
+
padding: 0 !important;
|
| 744 |
+
margin: 0 !important;
|
| 745 |
+
}
|
| 746 |
+
.view-help-host,
|
| 747 |
+
.view-help-host.block,
|
| 748 |
+
.view-help-host .padded,
|
| 749 |
+
.view-help-host .html-container,
|
| 750 |
+
.view-help-host .prose {
|
| 751 |
+
border: none !important;
|
| 752 |
+
background: transparent !important;
|
| 753 |
+
box-shadow: none !important;
|
| 754 |
+
padding: 0 !important;
|
| 755 |
+
margin: 0 0 10px !important;
|
| 756 |
+
}
|
| 757 |
+
.filter-help,
|
| 758 |
+
.prose .filter-help,
|
| 759 |
+
.view-help,
|
| 760 |
+
.prose .view-help {
|
| 761 |
+
margin: 0 !important;
|
| 762 |
+
color: var(--pruna-text-muted) !important;
|
| 763 |
+
font-size: 0.95rem !important;
|
| 764 |
+
line-height: 1.45 !important;
|
| 765 |
+
font-weight: 400 !important;
|
| 766 |
+
}
|
| 767 |
+
.view-filters span[data-testid="block-info"],
|
| 768 |
+
.view-filters .info,
|
| 769 |
+
.view-filters .block-info {
|
| 770 |
+
color: var(--pruna-text-muted) !important;
|
| 771 |
+
font-size: 0.75rem !important;
|
| 772 |
+
line-height: 1.35 !important;
|
| 773 |
+
margin-top: 4px !important;
|
| 774 |
}
|
| 775 |
+
.compare-prompt-block {
|
| 776 |
+
border: 1px solid var(--pruna-border) !important;
|
| 777 |
+
border-radius: 16px !important;
|
| 778 |
+
background: var(--pruna-card-bg) !important;
|
| 779 |
+
box-shadow: var(--pruna-card-shadow) !important;
|
| 780 |
+
}
|
| 781 |
+
|
| 782 |
+
/* Page footer — flat community strip + quiet citation accordion */
|
| 783 |
+
.page-footer-host,
|
| 784 |
+
.page-footer-host .html-container,
|
| 785 |
+
.page-footer-host .prose,
|
| 786 |
+
.page-footer-host.block,
|
| 787 |
+
.page-footer-host .padded {
|
| 788 |
+
border: none !important;
|
| 789 |
+
background: transparent !important;
|
| 790 |
+
box-shadow: none !important;
|
| 791 |
+
padding: 0 !important;
|
| 792 |
+
margin: 0 !important;
|
| 793 |
width: 100% !important;
|
| 794 |
+
max-width: none !important;
|
| 795 |
+
overflow: visible !important;
|
| 796 |
+
}
|
| 797 |
+
.page-footer {
|
| 798 |
+
margin: 32px 0 16px;
|
| 799 |
+
padding: 20px 0 0;
|
| 800 |
+
border-top: 1px solid var(--pruna-hairline);
|
| 801 |
+
}
|
| 802 |
+
.page-footer h3 {
|
| 803 |
+
margin: 0 0 10px !important;
|
| 804 |
+
font-size: 1.05rem !important;
|
| 805 |
+
font-weight: 600 !important;
|
| 806 |
+
letter-spacing: -0.015em !important;
|
| 807 |
+
text-transform: none !important;
|
| 808 |
+
color: var(--pruna-text-primary) !important;
|
| 809 |
}
|
| 810 |
+
.community-footer-links {
|
| 811 |
+
display: flex;
|
| 812 |
+
flex-wrap: wrap;
|
| 813 |
+
gap: 8px 16px;
|
| 814 |
+
align-items: center;
|
| 815 |
}
|
| 816 |
+
.community-footer-links a {
|
| 817 |
+
color: var(--pruna-text-muted) !important;
|
| 818 |
+
text-decoration: none !important;
|
| 819 |
+
font-size: 0.9rem !important;
|
| 820 |
+
font-weight: 500 !important;
|
| 821 |
}
|
| 822 |
+
.community-footer-links a:hover {
|
| 823 |
+
color: var(--pruna-link) !important;
|
|
|
|
|
|
|
|
|
|
| 824 |
}
|
| 825 |
+
|
| 826 |
+
/* Citation accordion — full-width control, not a section heading */
|
| 827 |
+
.citation-accordion {
|
| 828 |
+
margin: 0 0 28px !important;
|
| 829 |
+
}
|
| 830 |
+
.citation-accordion,
|
| 831 |
+
.citation-accordion.block {
|
| 832 |
+
border: 1px solid var(--pruna-accordion-border) !important;
|
| 833 |
+
border-radius: 10px !important;
|
| 834 |
+
background: var(--pruna-accordion-bg) !important;
|
| 835 |
+
box-shadow: none !important;
|
| 836 |
+
overflow: hidden !important;
|
| 837 |
+
}
|
| 838 |
+
.citation-accordion > .label-wrap,
|
| 839 |
+
.citation-accordion .label-wrap {
|
| 840 |
+
border: none !important;
|
| 841 |
+
background: transparent !important;
|
| 842 |
+
box-shadow: none !important;
|
| 843 |
+
padding: 11px 14px !important;
|
| 844 |
+
font-size: 0.92rem !important;
|
| 845 |
+
font-weight: 500 !important;
|
| 846 |
+
color: var(--pruna-text-primary) !important;
|
| 847 |
+
letter-spacing: -0.01em !important;
|
| 848 |
+
}
|
| 849 |
+
.citation-accordion > .label-wrap:hover,
|
| 850 |
+
.citation-accordion .label-wrap:hover {
|
| 851 |
+
background: var(--pruna-menu-hover-bg) !important;
|
| 852 |
+
}
|
| 853 |
+
.citation-accordion > .label-wrap span,
|
| 854 |
+
.citation-accordion .label-wrap span {
|
| 855 |
+
color: var(--pruna-text-primary) !important;
|
| 856 |
+
font-size: 0.92rem !important;
|
| 857 |
+
font-weight: 500 !important;
|
| 858 |
+
}
|
| 859 |
+
.citation-accordion .icon,
|
| 860 |
+
.citation-accordion .label-wrap .icon {
|
| 861 |
+
color: var(--pruna-text-muted) !important;
|
| 862 |
+
opacity: 0.9;
|
| 863 |
+
}
|
| 864 |
+
.citation-accordion .prose,
|
| 865 |
+
.citation-accordion .markdown,
|
| 866 |
+
.citation-accordion pre,
|
| 867 |
+
.citation-accordion code {
|
| 868 |
+
color: var(--pruna-text-muted) !important;
|
| 869 |
+
background: transparent !important;
|
| 870 |
+
border: none !important;
|
| 871 |
+
box-shadow: none !important;
|
| 872 |
+
}
|
| 873 |
+
.citation-accordion .wrap,
|
| 874 |
+
.citation-accordion > .wrap {
|
| 875 |
+
border-top: 1px solid var(--pruna-hairline) !important;
|
| 876 |
+
padding: 4px 14px 12px !important;
|
| 877 |
+
background: transparent !important;
|
| 878 |
+
}
|
| 879 |
+
.citation-accordion pre {
|
| 880 |
+
margin: 0 !important;
|
| 881 |
+
padding: 0 !important;
|
| 882 |
+
font-size: 0.8rem !important;
|
| 883 |
+
line-height: 1.55 !important;
|
| 884 |
+
overflow-x: auto;
|
| 885 |
}
|
| 886 |
+
|
| 887 |
+
.pareto-plot,
|
| 888 |
+
.pareto-plot.block,
|
| 889 |
+
.pareto-plot .plotly,
|
| 890 |
+
.pareto-plot .js-plotly-plot,
|
| 891 |
+
.pareto-plot .plot-container,
|
| 892 |
+
.pareto-plot .svg-container,
|
| 893 |
+
.pareto-plot .main-svg {
|
| 894 |
+
width: 100% !important;
|
| 895 |
+
max-width: 100% !important;
|
| 896 |
+
background: transparent !important;
|
| 897 |
+
border-color: var(--pruna-border) !important;
|
| 898 |
}
|
| 899 |
+
.pareto-plot .modebar { display: none !important; }
|
| 900 |
+
|
| 901 |
.leaderboard-controls {
|
| 902 |
+
display: flex !important;
|
| 903 |
+
flex-wrap: wrap !important;
|
| 904 |
+
align-items: end !important;
|
| 905 |
+
gap: 10px !important;
|
| 906 |
+
margin-bottom: 12px;
|
| 907 |
+
overflow: visible !important;
|
| 908 |
+
}
|
| 909 |
+
.leaderboard-controls > div {
|
| 910 |
+
min-width: 0 !important;
|
| 911 |
+
}
|
| 912 |
+
.leaderboard-controls > .block,
|
| 913 |
+
.leaderboard-controls .form,
|
| 914 |
+
.leaderboard-controls .block {
|
| 915 |
+
border: none !important;
|
| 916 |
+
background: transparent !important;
|
| 917 |
+
box-shadow: none !important;
|
| 918 |
+
padding: 0 !important;
|
| 919 |
+
--block-border-width: 0 !important;
|
| 920 |
+
}
|
| 921 |
+
.leaderboard-controls label {
|
| 922 |
+
color: var(--pruna-text-muted) !important;
|
| 923 |
+
font-size: 0.8rem !important;
|
| 924 |
+
font-weight: 500 !important;
|
| 925 |
+
}
|
| 926 |
+
.leaderboard-controls textarea {
|
| 927 |
+
min-height: 40px !important;
|
| 928 |
+
height: 40px !important;
|
| 929 |
+
padding: 0 12px !important;
|
| 930 |
+
border: 1px solid var(--pruna-input-border) !important;
|
| 931 |
+
border-radius: 10px !important;
|
| 932 |
+
background: var(--pruna-input-bg) !important;
|
| 933 |
+
box-shadow: none !important;
|
| 934 |
+
color: var(--pruna-text-primary) !important;
|
| 935 |
+
font-family: var(--pruna-font) !important;
|
| 936 |
+
font-size: 0.9rem !important;
|
| 937 |
+
line-height: 40px !important;
|
| 938 |
+
resize: none !important;
|
| 939 |
+
}
|
| 940 |
+
.leaderboard-controls textarea:focus {
|
| 941 |
+
border-color: var(--pruna-focus-border) !important;
|
| 942 |
+
outline: none !important;
|
| 943 |
+
box-shadow: var(--pruna-focus-ring) !important;
|
| 944 |
+
}
|
| 945 |
+
.leaderboard-controls .wrap {
|
| 946 |
+
min-height: 0 !important;
|
| 947 |
+
border: none !important;
|
| 948 |
+
background: transparent !important;
|
| 949 |
+
box-shadow: none !important;
|
| 950 |
+
overflow: visible !important;
|
| 951 |
+
}
|
| 952 |
+
.leaderboard-controls .wrap-inner {
|
| 953 |
+
display: flex !important;
|
| 954 |
+
flex-wrap: wrap !important;
|
| 955 |
+
align-items: center !important;
|
| 956 |
+
gap: 6px !important;
|
| 957 |
+
min-height: 40px !important;
|
| 958 |
+
height: auto !important;
|
| 959 |
+
padding: 4px 10px !important;
|
| 960 |
+
border: 1px solid var(--pruna-input-border) !important;
|
| 961 |
+
border-radius: 10px !important;
|
| 962 |
+
background: var(--pruna-input-bg) !important;
|
| 963 |
+
box-shadow: none !important;
|
| 964 |
+
overflow: visible !important;
|
| 965 |
+
}
|
| 966 |
+
.leaderboard-controls .secondary-wrap {
|
| 967 |
+
display: flex !important;
|
| 968 |
+
flex-wrap: wrap !important;
|
| 969 |
+
align-items: center !important;
|
| 970 |
+
gap: 6px !important;
|
| 971 |
+
flex: 1 1 auto !important;
|
| 972 |
+
min-width: 0 !important;
|
| 973 |
+
min-height: 0 !important;
|
| 974 |
+
height: auto !important;
|
| 975 |
+
margin: 0 !important;
|
| 976 |
+
padding: 0 !important;
|
| 977 |
+
border: none !important;
|
| 978 |
+
background: transparent !important;
|
| 979 |
+
box-shadow: none !important;
|
| 980 |
+
}
|
| 981 |
+
.leaderboard-controls .token {
|
| 982 |
+
display: inline-flex !important;
|
| 983 |
+
align-items: center !important;
|
| 984 |
+
max-width: 100% !important;
|
| 985 |
+
margin: 0 !important;
|
| 986 |
+
}
|
| 987 |
+
.leaderboard-controls .token:empty {
|
| 988 |
+
display: none !important;
|
| 989 |
}
|
| 990 |
+
.leaderboard-controls .dropdown-arrow,
|
| 991 |
+
.leaderboard-controls .icon-wrap {
|
| 992 |
+
flex: 0 0 auto !important;
|
| 993 |
+
align-self: center !important;
|
| 994 |
+
margin-left: auto !important;
|
| 995 |
+
}
|
| 996 |
+
.ranking-table-host,
|
| 997 |
+
.ranking-table-host .html-container,
|
| 998 |
+
.ranking-table-host .prose,
|
| 999 |
+
.ranking-table-host.block,
|
| 1000 |
+
.ranking-table-host .padded,
|
| 1001 |
+
.ranking-table-host.prose {
|
| 1002 |
+
display: grid !important;
|
| 1003 |
+
grid-template-columns: minmax(0, 1fr) !important;
|
| 1004 |
+
border: none !important;
|
| 1005 |
+
outline: none !important;
|
| 1006 |
+
background: transparent !important;
|
| 1007 |
+
box-shadow: none !important;
|
| 1008 |
+
padding: 0 !important;
|
| 1009 |
+
margin: 0 !important;
|
| 1010 |
+
width: 100% !important;
|
| 1011 |
+
max-width: 100% !important;
|
| 1012 |
+
min-width: 0 !important;
|
| 1013 |
+
overflow: hidden !important;
|
| 1014 |
+
--block-border-width: 0 !important;
|
| 1015 |
}
|
| 1016 |
.ranking-table-scroll {
|
| 1017 |
+
border: 1px solid var(--pruna-menu-border);
|
| 1018 |
+
border-radius: 12px;
|
| 1019 |
+
background: var(--pruna-menu-bg);
|
| 1020 |
+
box-shadow: none;
|
| 1021 |
+
width: 100%;
|
| 1022 |
+
max-width: 100%;
|
| 1023 |
+
min-width: 0;
|
| 1024 |
+
max-height: min(70vh, 720px);
|
| 1025 |
overflow-x: auto;
|
| 1026 |
+
overflow-y: auto;
|
| 1027 |
+
-webkit-overflow-scrolling: touch;
|
| 1028 |
+
overscroll-behavior-x: contain;
|
| 1029 |
}
|
| 1030 |
+
.ranking-table,
|
| 1031 |
+
.prose .ranking-table {
|
| 1032 |
width: 100%;
|
| 1033 |
+
margin: 0 !important;
|
| 1034 |
+
overflow: visible;
|
| 1035 |
+
}
|
| 1036 |
+
.ranking-table table,
|
| 1037 |
+
.prose .ranking-table table {
|
| 1038 |
+
display: table !important;
|
| 1039 |
+
width: max-content !important;
|
| 1040 |
+
min-width: 100% !important;
|
| 1041 |
+
max-width: none !important;
|
| 1042 |
+
margin: 0 !important;
|
| 1043 |
+
border-collapse: separate !important;
|
| 1044 |
+
border-spacing: 0 !important;
|
| 1045 |
+
color: var(--pruna-text-body) !important;
|
| 1046 |
+
font-size: 13px !important;
|
| 1047 |
+
line-height: 1.35 !important;
|
| 1048 |
+
background: transparent !important;
|
| 1049 |
+
border: none !important;
|
| 1050 |
+
outline: none !important;
|
| 1051 |
+
}
|
| 1052 |
+
.ranking-table thead { display: table-header-group !important; }
|
| 1053 |
+
.ranking-table tbody { display: table-row-group !important; }
|
| 1054 |
+
.ranking-table tr { display: table-row !important; }
|
| 1055 |
+
.ranking-table th,
|
| 1056 |
+
.ranking-table td,
|
| 1057 |
+
.prose .ranking-table th,
|
| 1058 |
+
.prose .ranking-table td {
|
| 1059 |
+
display: table-cell !important;
|
| 1060 |
+
vertical-align: middle !important;
|
| 1061 |
+
padding: 11px 14px !important;
|
| 1062 |
+
border: 0 !important;
|
| 1063 |
+
border-bottom: 1px solid var(--pruna-hairline) !important;
|
| 1064 |
+
background: transparent !important;
|
| 1065 |
+
box-shadow: none !important;
|
| 1066 |
+
outline: none !important;
|
| 1067 |
+
}
|
| 1068 |
+
.ranking-table th,
|
| 1069 |
+
.prose .ranking-table th {
|
| 1070 |
+
position: sticky;
|
| 1071 |
+
top: 0;
|
| 1072 |
+
z-index: 3;
|
| 1073 |
+
background: var(--pruna-bg-header) !important;
|
| 1074 |
+
color: var(--pruna-text-muted) !important;
|
| 1075 |
+
font-size: 11px !important;
|
| 1076 |
+
font-weight: 600 !important;
|
| 1077 |
letter-spacing: 0.04em;
|
| 1078 |
text-align: left;
|
| 1079 |
text-transform: uppercase;
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1080 |
white-space: nowrap;
|
| 1081 |
+
box-shadow: 0 1px 0 var(--pruna-hairline) !important;
|
| 1082 |
}
|
| 1083 |
+
.ranking-table th.sortable-col { cursor: pointer; user-select: none; }
|
| 1084 |
+
.ranking-table th.sortable-col:hover { color: var(--pruna-lavender) !important; }
|
|
|
|
| 1085 |
.ranking-table th.sortable-col.sorted-asc::after,
|
| 1086 |
.ranking-table th.sortable-col.sorted-desc::after {
|
| 1087 |
margin-left: 0.35em;
|
| 1088 |
font-size: 0.85em;
|
| 1089 |
opacity: 0.85;
|
| 1090 |
}
|
| 1091 |
+
.ranking-table th.sortable-col.sorted-asc::after { content: "▲"; }
|
| 1092 |
+
.ranking-table th.sortable-col.sorted-desc::after { content: "▼"; }
|
| 1093 |
+
.ranking-table tbody tr:last-child td { border-bottom: 0 !important; }
|
| 1094 |
+
.ranking-table tbody tr:hover td {
|
| 1095 |
+
background: var(--pruna-table-hover) !important;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1096 |
}
|
| 1097 |
.ranking-table tbody tr:hover td,
|
| 1098 |
.ranking-table tbody tr:hover .rank,
|
| 1099 |
.ranking-table tbody tr:hover .metric-score,
|
| 1100 |
+
.ranking-table tbody tr:hover .text-cell,
|
| 1101 |
.ranking-table tbody tr:hover .model-cell strong,
|
| 1102 |
+
.ranking-table tbody tr:hover .model-cell span { color: var(--pruna-table-hover-text) !important; }
|
| 1103 |
+
.ranking-table .rank,
|
| 1104 |
+
.prose .ranking-table .rank {
|
| 1105 |
+
position: sticky;
|
| 1106 |
+
left: 0;
|
| 1107 |
+
z-index: 1;
|
| 1108 |
+
box-sizing: border-box;
|
| 1109 |
+
width: 3.25rem;
|
| 1110 |
+
min-width: 3.25rem;
|
| 1111 |
+
color: var(--pruna-lavender) !important;
|
| 1112 |
+
font-weight: 700 !important;
|
| 1113 |
+
text-align: center;
|
| 1114 |
+
background: var(--pruna-table-sticky) !important;
|
| 1115 |
}
|
| 1116 |
+
.ranking-table th.rank,
|
| 1117 |
+
.prose .ranking-table th.rank {
|
| 1118 |
+
top: 0;
|
| 1119 |
+
left: 0;
|
| 1120 |
+
z-index: 5;
|
| 1121 |
+
text-align: center;
|
| 1122 |
+
background: var(--pruna-bg-header) !important;
|
| 1123 |
+
}
|
| 1124 |
+
.ranking-table .model-cell,
|
| 1125 |
+
.prose .ranking-table .model-cell {
|
| 1126 |
+
position: sticky;
|
| 1127 |
+
left: 3.25rem;
|
| 1128 |
+
z-index: 1;
|
| 1129 |
+
min-width: 180px;
|
| 1130 |
+
max-width: 260px;
|
| 1131 |
+
background: var(--pruna-table-sticky) !important;
|
| 1132 |
+
}
|
| 1133 |
+
.ranking-table th.model-cell,
|
| 1134 |
+
.prose .ranking-table th.model-cell {
|
| 1135 |
+
top: 0;
|
| 1136 |
+
left: 3.25rem;
|
| 1137 |
+
z-index: 5;
|
| 1138 |
+
background: var(--pruna-bg-header) !important;
|
| 1139 |
+
}
|
| 1140 |
+
.ranking-table tbody tr:hover .rank,
|
| 1141 |
+
.ranking-table tbody tr:hover .model-cell {
|
| 1142 |
+
background: var(--pruna-table-sticky-hover) !important;
|
| 1143 |
}
|
| 1144 |
.ranking-table .model-cell strong,
|
| 1145 |
+
.ranking-table .model-cell span { display: block; }
|
| 1146 |
+
.ranking-table .model-cell strong {
|
| 1147 |
+
color: var(--pruna-text-primary) !important;
|
| 1148 |
+
font-weight: 600 !important;
|
| 1149 |
+
white-space: nowrap;
|
| 1150 |
+
overflow: hidden;
|
| 1151 |
+
text-overflow: ellipsis;
|
| 1152 |
}
|
| 1153 |
.ranking-table .model-cell span {
|
| 1154 |
margin-top: 3px;
|
| 1155 |
+
color: var(--pruna-text-muted) !important;
|
| 1156 |
font-size: 12px;
|
| 1157 |
}
|
| 1158 |
+
.ranking-table .text-cell,
|
| 1159 |
+
.prose .ranking-table .text-cell {
|
| 1160 |
+
color: var(--pruna-text-muted) !important;
|
| 1161 |
+
font-weight: 500 !important;
|
| 1162 |
+
white-space: nowrap;
|
| 1163 |
+
}
|
| 1164 |
+
.ranking-table .metric-score,
|
| 1165 |
+
.prose .ranking-table .metric-score {
|
| 1166 |
+
color: var(--pruna-text-primary) !important;
|
| 1167 |
font-variant-numeric: tabular-nums;
|
| 1168 |
+
font-weight: 600 !important;
|
| 1169 |
+
text-align: right;
|
| 1170 |
+
white-space: nowrap;
|
| 1171 |
}
|
| 1172 |
+
.ranking-table th.metric-score { text-align: right; }
|
| 1173 |
.ranking-table .empty-state {
|
| 1174 |
+
color: var(--pruna-text-muted);
|
| 1175 |
padding: 32px;
|
| 1176 |
text-align: center;
|
| 1177 |
}
|
| 1178 |
+
|
| 1179 |
+
.compare-samples-help { color: var(--pruna-text-body); margin-bottom: 8px; }
|
| 1180 |
+
.compare-controls,
|
| 1181 |
+
.compare-controls.row,
|
| 1182 |
+
.compare-controls.column,
|
| 1183 |
+
.compare-controls .form {
|
| 1184 |
+
display: flex !important;
|
| 1185 |
+
flex-direction: column !important;
|
| 1186 |
+
flex-wrap: nowrap !important;
|
| 1187 |
+
align-items: stretch !important;
|
| 1188 |
+
align-content: flex-start !important;
|
| 1189 |
+
justify-content: flex-start !important;
|
| 1190 |
+
gap: 12px !important;
|
| 1191 |
+
width: 100% !important;
|
| 1192 |
+
max-width: 100% !important;
|
| 1193 |
+
flex-grow: 0 !important;
|
| 1194 |
+
height: auto !important;
|
| 1195 |
+
min-height: 0 !important;
|
| 1196 |
+
margin-bottom: 16px;
|
| 1197 |
+
}
|
| 1198 |
+
.compare-controls > .block,
|
| 1199 |
+
.compare-controls .form,
|
| 1200 |
+
.compare-controls .block,
|
| 1201 |
+
.compare-controls .compare-models,
|
| 1202 |
+
.compare-controls .compare-prompt-count {
|
| 1203 |
+
border: none !important;
|
| 1204 |
+
background: transparent !important;
|
| 1205 |
+
box-shadow: none !important;
|
| 1206 |
+
padding: 0 !important;
|
| 1207 |
+
--block-border-width: 0 !important;
|
| 1208 |
+
height: auto !important;
|
| 1209 |
+
min-height: 0 !important;
|
| 1210 |
+
}
|
| 1211 |
+
.compare-controls label,
|
| 1212 |
+
.compare-controls span[data-testid="block-info"] {
|
| 1213 |
+
display: block !important;
|
| 1214 |
+
margin: 0 0 6px !important;
|
| 1215 |
+
padding: 0 !important;
|
| 1216 |
+
color: var(--pruna-text-muted) !important;
|
| 1217 |
+
font-size: 0.8rem !important;
|
| 1218 |
+
font-weight: 500 !important;
|
| 1219 |
+
line-height: 1.2 !important;
|
| 1220 |
+
}
|
| 1221 |
+
.compare-controls .compare-models span[data-testid="block-info"] {
|
| 1222 |
+
min-height: 1.15rem;
|
| 1223 |
+
}
|
| 1224 |
+
.compare-controls .compare-models {
|
| 1225 |
+
flex: 0 0 auto !important;
|
| 1226 |
+
width: 100% !important;
|
| 1227 |
+
min-width: 0 !important;
|
| 1228 |
+
max-width: 100% !important;
|
| 1229 |
+
align-self: stretch !important;
|
| 1230 |
}
|
| 1231 |
+
.compare-controls .compare-prompt-count {
|
| 1232 |
+
flex: 0 0 auto !important;
|
| 1233 |
+
width: 100% !important;
|
| 1234 |
+
min-width: 0 !important;
|
| 1235 |
+
max-width: 100% !important;
|
| 1236 |
+
align-self: stretch !important;
|
| 1237 |
}
|
| 1238 |
+
.compare-controls .compare-shuffle {
|
| 1239 |
+
flex: 0 0 auto !important;
|
| 1240 |
+
width: 100% !important;
|
| 1241 |
+
max-width: 100% !important;
|
| 1242 |
+
align-self: stretch !important;
|
| 1243 |
+
}
|
| 1244 |
+
.compare-controls .compare-models .wrap,
|
| 1245 |
+
.compare-controls .compare-models .wrap-inner {
|
| 1246 |
+
min-height: 40px !important;
|
| 1247 |
+
height: auto !important;
|
| 1248 |
+
overflow: hidden !important;
|
| 1249 |
+
border: 1px solid var(--pruna-input-border) !important;
|
| 1250 |
+
border-radius: 10px !important;
|
| 1251 |
+
background: var(--pruna-input-bg) !important;
|
| 1252 |
+
box-shadow: none !important;
|
| 1253 |
+
}
|
| 1254 |
+
.compare-controls .compare-models .wrap-inner {
|
| 1255 |
+
display: flex !important;
|
| 1256 |
+
flex-direction: row !important;
|
| 1257 |
+
flex-wrap: wrap !important;
|
| 1258 |
+
align-items: center !important;
|
| 1259 |
+
gap: 6px !important;
|
| 1260 |
+
width: 100% !important;
|
| 1261 |
+
padding: 4px 8px !important;
|
| 1262 |
+
}
|
| 1263 |
+
.compare-controls .compare-models .secondary-wrap {
|
| 1264 |
+
display: flex !important;
|
| 1265 |
+
flex-direction: row !important;
|
| 1266 |
+
flex-wrap: wrap !important;
|
| 1267 |
+
align-items: center !important;
|
| 1268 |
+
gap: 6px !important;
|
| 1269 |
+
flex: 1 1 auto !important;
|
| 1270 |
+
width: auto !important;
|
| 1271 |
+
min-width: 0 !important;
|
| 1272 |
+
height: auto !important;
|
| 1273 |
+
margin: 0 !important;
|
| 1274 |
+
padding: 0 !important;
|
| 1275 |
+
border: none !important;
|
| 1276 |
+
background: transparent !important;
|
| 1277 |
+
box-shadow: none !important;
|
| 1278 |
+
}
|
| 1279 |
+
.compare-controls .compare-models .token {
|
| 1280 |
+
display: inline-flex !important;
|
| 1281 |
+
align-items: center !important;
|
| 1282 |
+
justify-content: center !important;
|
| 1283 |
+
width: auto !important;
|
| 1284 |
+
max-width: 100% !important;
|
| 1285 |
+
margin: 0 !important;
|
| 1286 |
+
white-space: nowrap !important;
|
| 1287 |
+
}
|
| 1288 |
+
.compare-controls .compare-models .token:empty {
|
| 1289 |
+
display: none !important;
|
| 1290 |
}
|
| 1291 |
+
.compare-controls .compare-models input {
|
| 1292 |
+
width: auto !important;
|
| 1293 |
+
min-width: 2.5rem !important;
|
| 1294 |
+
flex: 1 1 5rem !important;
|
| 1295 |
+
height: 28px !important;
|
| 1296 |
+
min-height: 28px !important;
|
| 1297 |
+
}
|
| 1298 |
+
.compare-controls .compare-models .dropdown-arrow,
|
| 1299 |
+
.compare-controls .compare-models .icon-wrap {
|
| 1300 |
+
flex: 0 0 auto !important;
|
| 1301 |
+
align-self: center !important;
|
| 1302 |
+
margin-left: auto !important;
|
| 1303 |
+
}
|
| 1304 |
+
.compare-controls .compare-prompt-count .wrap {
|
| 1305 |
+
display: grid !important;
|
| 1306 |
+
grid-template-columns: minmax(0, 1fr) auto;
|
| 1307 |
+
grid-template-rows: auto 40px;
|
| 1308 |
+
gap: 6px 10px;
|
| 1309 |
+
align-items: center;
|
| 1310 |
+
width: 100%;
|
| 1311 |
+
min-height: 0 !important;
|
| 1312 |
+
height: auto !important;
|
| 1313 |
+
margin: 0 !important;
|
| 1314 |
+
padding: 0 !important;
|
| 1315 |
+
border: none !important;
|
| 1316 |
+
background: transparent !important;
|
| 1317 |
+
box-shadow: none !important;
|
| 1318 |
+
}
|
| 1319 |
+
.compare-controls .compare-prompt-count .head {
|
| 1320 |
+
display: contents;
|
| 1321 |
+
margin: 0 !important;
|
| 1322 |
+
}
|
| 1323 |
+
.compare-controls .compare-prompt-count .head label {
|
| 1324 |
+
grid-column: 1 / -1;
|
| 1325 |
+
grid-row: 1;
|
| 1326 |
+
}
|
| 1327 |
+
.compare-controls .compare-prompt-count .tab-like-container {
|
| 1328 |
+
grid-column: 2;
|
| 1329 |
+
grid-row: 2;
|
| 1330 |
+
height: 40px !important;
|
| 1331 |
+
min-height: 40px !important;
|
| 1332 |
+
align-self: stretch;
|
| 1333 |
+
border: 1px solid var(--pruna-input-border) !important;
|
| 1334 |
+
border-radius: 10px !important;
|
| 1335 |
+
background: var(--pruna-input-bg) !important;
|
| 1336 |
+
overflow: hidden;
|
| 1337 |
+
}
|
| 1338 |
+
.compare-controls .compare-prompt-count .slider_input_container {
|
| 1339 |
+
grid-column: 1;
|
| 1340 |
+
grid-row: 2;
|
| 1341 |
+
height: 40px !important;
|
| 1342 |
+
min-height: 40px !important;
|
| 1343 |
+
margin: 0 !important;
|
| 1344 |
+
gap: 0 !important;
|
| 1345 |
+
}
|
| 1346 |
+
.compare-controls .compare-prompt-count .min_value,
|
| 1347 |
+
.compare-controls .compare-prompt-count .max_value {
|
| 1348 |
+
display: none !important;
|
| 1349 |
}
|
| 1350 |
+
.compare-controls .compare-prompt-count input[type="range"] {
|
| 1351 |
+
height: 40px !important;
|
| 1352 |
+
min-width: 0 !important;
|
| 1353 |
+
margin: 0 !important;
|
| 1354 |
+
padding: 0 !important;
|
| 1355 |
+
background: transparent !important;
|
| 1356 |
+
border: none !important;
|
| 1357 |
+
box-shadow: none !important;
|
| 1358 |
+
}
|
| 1359 |
+
.compare-controls .compare-prompt-count input[type="number"] {
|
| 1360 |
+
height: 40px !important;
|
| 1361 |
+
min-height: 40px !important;
|
| 1362 |
+
min-width: 2.5rem !important;
|
| 1363 |
+
margin: 0 !important;
|
| 1364 |
+
padding: 0 8px !important;
|
| 1365 |
+
line-height: 40px !important;
|
| 1366 |
+
border: none !important;
|
| 1367 |
+
background: transparent !important;
|
| 1368 |
+
box-shadow: none !important;
|
| 1369 |
+
}
|
| 1370 |
+
.compare-controls .compare-shuffle.block:not(button) {
|
| 1371 |
+
width: 100% !important;
|
| 1372 |
+
max-width: 100% !important;
|
| 1373 |
+
align-self: stretch !important;
|
| 1374 |
+
background: transparent !important;
|
| 1375 |
+
border: none !important;
|
| 1376 |
+
box-shadow: none !important;
|
| 1377 |
+
padding: 0 !important;
|
| 1378 |
+
}
|
| 1379 |
+
.compare-controls button.compare-shuffle,
|
| 1380 |
+
.compare-controls .compare-shuffle button,
|
| 1381 |
+
.compare-controls .compare-shuffle button.primary,
|
| 1382 |
+
.compare-controls .compare-shuffle button.secondary {
|
| 1383 |
+
display: inline-flex !important;
|
| 1384 |
+
justify-content: center !important;
|
| 1385 |
+
align-items: center !important;
|
| 1386 |
+
width: 100% !important;
|
| 1387 |
+
min-width: 0 !important;
|
| 1388 |
+
height: 44px !important;
|
| 1389 |
+
min-height: 44px !important;
|
| 1390 |
+
margin: 0 !important;
|
| 1391 |
+
padding: 0 16px !important;
|
| 1392 |
+
border-radius: 10px !important;
|
| 1393 |
+
align-self: stretch !important;
|
| 1394 |
+
appearance: none !important;
|
| 1395 |
+
-webkit-appearance: none !important;
|
| 1396 |
+
-webkit-tap-highlight-color: transparent !important;
|
| 1397 |
+
background: var(--pruna-accent) !important;
|
| 1398 |
+
border: 1px solid color-mix(in oklab, var(--pruna-accent) 72%, transparent) !important;
|
| 1399 |
+
color: #fff !important;
|
| 1400 |
+
font-family: var(--pruna-font) !important;
|
| 1401 |
+
font-size: 0.95rem !important;
|
| 1402 |
+
font-weight: 600 !important;
|
| 1403 |
+
letter-spacing: -0.01em !important;
|
| 1404 |
+
box-shadow: inset 0 1px #fff3 !important;
|
| 1405 |
+
outline: none !important;
|
| 1406 |
+
}
|
| 1407 |
+
.compare-controls button.compare-shuffle:hover,
|
| 1408 |
+
.compare-controls button.compare-shuffle:active,
|
| 1409 |
+
.compare-controls button.compare-shuffle:focus,
|
| 1410 |
+
.compare-controls button.compare-shuffle:focus-visible,
|
| 1411 |
+
.compare-controls .compare-shuffle button:hover,
|
| 1412 |
+
.compare-controls .compare-shuffle button:active,
|
| 1413 |
+
.compare-controls .compare-shuffle button:focus,
|
| 1414 |
+
.compare-controls .compare-shuffle button:focus-visible {
|
| 1415 |
+
background: var(--pruna-accent) !important;
|
| 1416 |
+
border-color: color-mix(in oklab, var(--pruna-accent) 72%, transparent) !important;
|
| 1417 |
+
color: #fff !important;
|
| 1418 |
+
outline: none !important;
|
| 1419 |
+
box-shadow: inset 0 1px #fff3 !important;
|
| 1420 |
+
-webkit-tap-highlight-color: transparent !important;
|
| 1421 |
+
}
|
| 1422 |
+
@media (min-width: 701px) {
|
| 1423 |
+
.compare-controls,
|
| 1424 |
+
.compare-controls.row,
|
| 1425 |
+
.compare-controls.column,
|
| 1426 |
+
.compare-controls .form {
|
| 1427 |
+
flex-direction: row !important;
|
| 1428 |
+
flex-wrap: nowrap !important;
|
| 1429 |
+
align-items: flex-end !important;
|
| 1430 |
+
}
|
| 1431 |
+
.compare-controls .compare-models {
|
| 1432 |
+
flex: 4 1 0 !important;
|
| 1433 |
+
width: auto !important;
|
| 1434 |
+
max-width: none !important;
|
| 1435 |
+
align-self: flex-end !important;
|
| 1436 |
+
}
|
| 1437 |
+
.compare-controls .compare-prompt-count {
|
| 1438 |
+
flex: 1 1 200px !important;
|
| 1439 |
+
width: auto !important;
|
| 1440 |
+
min-width: 180px !important;
|
| 1441 |
+
max-width: 280px !important;
|
| 1442 |
+
align-self: flex-end !important;
|
| 1443 |
+
}
|
| 1444 |
+
.compare-controls .compare-shuffle,
|
| 1445 |
+
.compare-controls .compare-shuffle.block:not(button) {
|
| 1446 |
+
flex: 0 0 auto !important;
|
| 1447 |
+
width: auto !important;
|
| 1448 |
+
max-width: none !important;
|
| 1449 |
+
align-self: flex-end !important;
|
| 1450 |
+
}
|
| 1451 |
+
.compare-controls button.compare-shuffle,
|
| 1452 |
+
.compare-controls .compare-shuffle button {
|
| 1453 |
+
width: auto !important;
|
| 1454 |
+
min-width: 10.5rem !important;
|
| 1455 |
+
}
|
| 1456 |
}
|
| 1457 |
.compare-prompt-block {
|
| 1458 |
margin: 0 0 22px;
|
| 1459 |
padding: 14px 16px;
|
| 1460 |
+
background: var(--pruna-callout-bg) !important;
|
|
|
|
|
|
|
| 1461 |
}
|
| 1462 |
.compare-prompt-meta {
|
| 1463 |
display: flex;
|
| 1464 |
justify-content: space-between;
|
| 1465 |
gap: 12px;
|
| 1466 |
margin-bottom: 8px;
|
| 1467 |
+
color: var(--pruna-text-muted);
|
| 1468 |
font-size: 12px;
|
| 1469 |
font-weight: 700;
|
| 1470 |
letter-spacing: 0.04em;
|
|
|
|
| 1472 |
}
|
| 1473 |
.compare-prompt-text {
|
| 1474 |
margin: 0 0 14px;
|
| 1475 |
+
color: var(--pruna-text-body);
|
| 1476 |
font-size: 15px;
|
| 1477 |
line-height: 1.45;
|
| 1478 |
}
|
| 1479 |
+
.compare-gallery,
|
| 1480 |
+
.compare-gallery.block,
|
| 1481 |
+
.compare-gallery .html-container,
|
| 1482 |
+
.compare-gallery .prose {
|
| 1483 |
+
width: 100% !important;
|
| 1484 |
+
max-width: 100% !important;
|
| 1485 |
+
min-width: 0 !important;
|
| 1486 |
+
overflow-x: hidden !important;
|
| 1487 |
+
border: none !important;
|
| 1488 |
+
background: transparent !important;
|
| 1489 |
+
box-shadow: none !important;
|
| 1490 |
+
padding: 0 !important;
|
| 1491 |
+
margin: 0 !important;
|
| 1492 |
+
}
|
| 1493 |
+
.compare-row { display: grid; gap: 12px; min-width: 0; width: 100%; }
|
| 1494 |
+
.compare-prompt-text { overflow-wrap: anywhere; }
|
| 1495 |
+
.pareto-layout,
|
| 1496 |
+
.pareto-layout.row,
|
| 1497 |
+
.pareto-layout .form {
|
| 1498 |
+
display: flex !important;
|
| 1499 |
+
flex-wrap: wrap !important;
|
| 1500 |
+
align-items: stretch !important;
|
| 1501 |
+
gap: 16px !important;
|
| 1502 |
+
width: 100% !important;
|
| 1503 |
+
max-width: 100% !important;
|
| 1504 |
}
|
| 1505 |
+
.pareto-layout > .pareto-col,
|
| 1506 |
+
.pareto-layout .form > .pareto-col {
|
| 1507 |
+
flex: 1 1 360px !important;
|
| 1508 |
+
min-width: 0 !important;
|
| 1509 |
+
max-width: 100% !important;
|
| 1510 |
}
|
| 1511 |
+
.compare-cell { min-width: 0; }
|
| 1512 |
.compare-model-label {
|
| 1513 |
margin-bottom: 6px;
|
| 1514 |
+
color: var(--pruna-lavender);
|
| 1515 |
font-size: 13px;
|
| 1516 |
font-weight: 700;
|
| 1517 |
word-break: break-word;
|
|
|
|
| 1521 |
width: 100%;
|
| 1522 |
aspect-ratio: 1 / 1;
|
| 1523 |
object-fit: cover;
|
| 1524 |
+
border-radius: 12px;
|
| 1525 |
+
border: 1px solid var(--pruna-border);
|
| 1526 |
+
background: var(--pruna-bg-elevated);
|
| 1527 |
}
|
| 1528 |
+
.compare-empty,
|
| 1529 |
+
.pareto-note-copy {
|
| 1530 |
+
margin: 0;
|
| 1531 |
+
padding: 14px 16px;
|
| 1532 |
+
background: var(--pruna-callout-bg, var(--pruna-bg-card));
|
| 1533 |
+
border: 1px dashed color-mix(in oklab, var(--pruna-border) 70%, transparent);
|
| 1534 |
border-radius: 10px;
|
| 1535 |
+
color: var(--pruna-text-muted);
|
| 1536 |
+
font-size: 0.95rem;
|
| 1537 |
+
line-height: 1.5;
|
| 1538 |
+
text-align: left;
|
| 1539 |
}
|
| 1540 |
+
|
| 1541 |
+
/*
|
| 1542 |
+
Type scale (playground-like):
|
| 1543 |
+
product title > page h1 > section h2/h3 > meta labels > body
|
| 1544 |
+
Uppercase lavender is reserved for meta labels only — not section titles.
|
| 1545 |
+
*/
|
| 1546 |
+
.app-header-brand h1,
|
| 1547 |
+
.prose .app-header-brand h1,
|
| 1548 |
+
.gradio-container .app-header-brand h1,
|
| 1549 |
+
.app-header .app-header-brand h1 {
|
| 1550 |
+
display: block !important;
|
| 1551 |
+
width: max-content !important;
|
| 1552 |
+
max-width: none !important;
|
| 1553 |
+
flex: 0 0 auto !important;
|
| 1554 |
+
margin: 0 !important;
|
| 1555 |
+
padding: 0 !important;
|
| 1556 |
+
text-align: center !important;
|
| 1557 |
+
font-size: clamp(1.9rem, 3vw, 2.4rem) !important;
|
| 1558 |
+
font-weight: 700 !important;
|
| 1559 |
+
letter-spacing: -0.035em;
|
| 1560 |
+
line-height: 1 !important;
|
| 1561 |
+
color: var(--pruna-text-primary) !important;
|
| 1562 |
+
}
|
| 1563 |
+
.app-header-tagline,
|
| 1564 |
+
.prose .app-header-tagline,
|
| 1565 |
+
.gradio-container .app-header-tagline,
|
| 1566 |
+
.prose p.app-header-tagline {
|
| 1567 |
+
display: block !important;
|
| 1568 |
+
width: 100% !important;
|
| 1569 |
+
margin: 8px 0 0 !important;
|
| 1570 |
+
padding: 0 !important;
|
| 1571 |
+
text-align: center !important;
|
| 1572 |
+
color: var(--pruna-text-muted) !important;
|
| 1573 |
+
font-size: 0.95rem !important;
|
| 1574 |
+
line-height: 1.45 !important;
|
| 1575 |
+
font-weight: 400 !important;
|
| 1576 |
}
|
| 1577 |
+
|
| 1578 |
+
.gradio-container h1,
|
| 1579 |
+
.markdown h1, .md h1, .prose h1 {
|
| 1580 |
+
margin: 0 0 0.65rem !important;
|
| 1581 |
+
padding: 0 !important;
|
| 1582 |
+
color: var(--pruna-text-primary) !important;
|
| 1583 |
+
font-family: var(--pruna-font) !important;
|
| 1584 |
+
font-size: 1.5rem !important;
|
| 1585 |
+
font-weight: 700 !important;
|
| 1586 |
+
letter-spacing: -0.03em !important;
|
| 1587 |
+
line-height: 1.2 !important;
|
| 1588 |
+
text-transform: none !important;
|
| 1589 |
+
}
|
| 1590 |
+
.gradio-container h2,
|
| 1591 |
+
.markdown h2, .md h2, .prose h2 {
|
| 1592 |
+
margin: 1.75rem 0 0.75rem !important;
|
| 1593 |
+
padding: 0 !important;
|
| 1594 |
+
color: var(--pruna-text-primary) !important;
|
| 1595 |
+
font-family: var(--pruna-font) !important;
|
| 1596 |
+
font-size: 1.25rem !important;
|
| 1597 |
+
font-weight: 600 !important;
|
| 1598 |
+
letter-spacing: -0.02em !important;
|
| 1599 |
+
line-height: 1.3 !important;
|
| 1600 |
+
text-transform: none !important;
|
| 1601 |
+
}
|
| 1602 |
+
.gradio-container h3,
|
| 1603 |
+
.markdown h3, .md h3, .prose h3 {
|
| 1604 |
+
margin: 1.35rem 0 0.5rem !important;
|
| 1605 |
+
padding: 0 !important;
|
| 1606 |
+
color: var(--pruna-text-primary) !important;
|
| 1607 |
+
font-family: var(--pruna-font) !important;
|
| 1608 |
+
font-size: 1.05rem !important;
|
| 1609 |
+
font-weight: 600 !important;
|
| 1610 |
+
letter-spacing: -0.015em !important;
|
| 1611 |
+
line-height: 1.35 !important;
|
| 1612 |
+
text-transform: none !important;
|
| 1613 |
+
}
|
| 1614 |
+
.gradio-container h4,
|
| 1615 |
+
.markdown h4, .md h4, .prose h4 {
|
| 1616 |
+
margin: 1rem 0 0.4rem !important;
|
| 1617 |
+
padding: 0 !important;
|
| 1618 |
+
color: var(--pruna-text-body) !important;
|
| 1619 |
+
font-family: var(--pruna-font) !important;
|
| 1620 |
+
font-size: 0.95rem !important;
|
| 1621 |
+
font-weight: 600 !important;
|
| 1622 |
+
letter-spacing: -0.01em !important;
|
| 1623 |
+
line-height: 1.35 !important;
|
| 1624 |
+
text-transform: none !important;
|
| 1625 |
}
|
| 1626 |
+
|
| 1627 |
+
.markdown, .md, .prose,
|
| 1628 |
+
.gradio-container .markdown,
|
| 1629 |
+
.gradio-container .prose {
|
| 1630 |
+
overflow-wrap: break-word;
|
| 1631 |
+
max-width: 100%;
|
| 1632 |
}
|
| 1633 |
+
.markdown p, .md p, .prose p,
|
| 1634 |
+
.gradio-container .markdown p {
|
| 1635 |
+
margin: 0 0 0.8rem !important;
|
| 1636 |
+
color: var(--pruna-text-body) !important;
|
| 1637 |
+
font-size: 0.95rem !important;
|
| 1638 |
+
font-weight: 400 !important;
|
| 1639 |
+
line-height: 1.55 !important;
|
| 1640 |
}
|
| 1641 |
+
.markdown li, .md li, .prose li {
|
| 1642 |
+
color: var(--pruna-text-body) !important;
|
| 1643 |
+
font-size: 0.95rem;
|
| 1644 |
+
line-height: 1.5;
|
| 1645 |
}
|
| 1646 |
+
.markdown strong, .md strong, .prose strong {
|
| 1647 |
+
color: var(--pruna-text-primary) !important;
|
| 1648 |
+
font-weight: 600 !important;
|
|
|
|
| 1649 |
}
|
| 1650 |
+
|
| 1651 |
+
label, .block-label span {
|
| 1652 |
+
color: var(--pruna-text-muted) !important;
|
| 1653 |
+
font-size: 0.8rem !important;
|
| 1654 |
+
font-weight: 500 !important;
|
| 1655 |
+
}
|
| 1656 |
+
a { color: var(--pruna-link); }
|
| 1657 |
+
|
| 1658 |
+
/* Gradio form chrome → playground */
|
| 1659 |
+
button.primary, .gr-button-primary, .primary-button,
|
| 1660 |
+
button.primary:hover, button.primary:active, button.primary:focus,
|
| 1661 |
+
.gr-button-primary:hover, .gr-button-primary:active {
|
| 1662 |
+
background: var(--pruna-accent) !important;
|
| 1663 |
+
border: 1px solid color-mix(in oklab, var(--pruna-accent) 72%, transparent) !important;
|
| 1664 |
+
color: #fff !important;
|
| 1665 |
+
border-radius: 8px !important;
|
| 1666 |
+
box-shadow: inset 0 1px #fff3 !important;
|
| 1667 |
+
font-weight: 600 !important;
|
| 1668 |
+
outline: none !important;
|
| 1669 |
+
-webkit-tap-highlight-color: transparent !important;
|
| 1670 |
+
}
|
| 1671 |
+
button.secondary:active, button.secondary:focus,
|
| 1672 |
+
.gr-button-secondary:active {
|
| 1673 |
+
outline: none !important;
|
| 1674 |
+
-webkit-tap-highlight-color: transparent !important;
|
| 1675 |
+
}
|
| 1676 |
+
.block, .gr-group, .form, .panel {
|
| 1677 |
+
border-color: var(--pruna-border) !important;
|
| 1678 |
+
background: transparent !important;
|
| 1679 |
+
}
|
| 1680 |
+
input, textarea, select {
|
| 1681 |
+
background: var(--pruna-bg-card) !important;
|
| 1682 |
+
border-color: var(--pruna-border) !important;
|
| 1683 |
+
color: var(--pruna-text-primary) !important;
|
| 1684 |
+
border-radius: 10px !important;
|
| 1685 |
+
}
|
| 1686 |
+
.view-filters input,
|
| 1687 |
+
.leaderboard-controls .wrap-inner input {
|
| 1688 |
+
background: transparent !important;
|
| 1689 |
+
border: none !important;
|
| 1690 |
+
box-shadow: none !important;
|
| 1691 |
+
min-height: 28px !important;
|
| 1692 |
+
height: 28px !important;
|
| 1693 |
+
padding: 0 4px !important;
|
| 1694 |
+
line-height: 28px !important;
|
| 1695 |
+
}
|
| 1696 |
+
.view-filters .wrap-inner:focus-within,
|
| 1697 |
+
.leaderboard-controls .wrap-inner:focus-within {
|
| 1698 |
+
border-color: var(--pruna-focus-border) !important;
|
| 1699 |
+
box-shadow: var(--pruna-focus-ring) !important;
|
| 1700 |
+
}
|
| 1701 |
+
footer, .footer { display: none !important; }
|
| 1702 |
+
|
| 1703 |
+
.pareto-note,
|
| 1704 |
+
.pareto-note.block,
|
| 1705 |
+
.pareto-note .html-container,
|
| 1706 |
+
.pareto-note .prose {
|
| 1707 |
+
border: none !important;
|
| 1708 |
+
background: transparent !important;
|
| 1709 |
+
box-shadow: none !important;
|
| 1710 |
+
padding: 0 !important;
|
| 1711 |
+
margin: 0 0 12px !important;
|
| 1712 |
+
}
|
| 1713 |
+
.pareto-metric-block {
|
| 1714 |
+
padding-bottom: 0.5rem;
|
| 1715 |
+
border-bottom: 1px solid var(--pruna-hairline);
|
| 1716 |
+
}
|
| 1717 |
+
.pareto-metric-block:last-child {
|
| 1718 |
+
border-bottom: none;
|
| 1719 |
+
}
|
| 1720 |
+
.pareto-metric-title {
|
| 1721 |
+
margin: 0 0 0.75rem;
|
| 1722 |
+
color: var(--pruna-text-primary);
|
| 1723 |
+
font-size: 1.05rem;
|
| 1724 |
+
font-weight: 600;
|
| 1725 |
+
}
|
| 1726 |
+
.pareto-subhead,
|
| 1727 |
+
.pareto-subhead.block,
|
| 1728 |
+
.pareto-subhead .prose,
|
| 1729 |
+
.pareto-subhead .html-container {
|
| 1730 |
+
border: none !important;
|
| 1731 |
+
background: transparent !important;
|
| 1732 |
+
box-shadow: none !important;
|
| 1733 |
+
padding: 0 !important;
|
| 1734 |
+
margin: 0 0 0.4rem !important;
|
| 1735 |
+
color: var(--pruna-text-muted);
|
| 1736 |
+
}
|
| 1737 |
+
.pareto-subhead h4,
|
| 1738 |
+
.pareto-subhead .prose h4 {
|
| 1739 |
+
margin: 0 !important;
|
| 1740 |
+
color: var(--pruna-text-muted) !important;
|
| 1741 |
+
font-size: 0.95rem !important;
|
| 1742 |
+
font-weight: 600 !important;
|
| 1743 |
+
}
|
| 1744 |
+
|
| 1745 |
+
.options,
|
| 1746 |
+
ul.options {
|
| 1747 |
+
position: absolute !important;
|
| 1748 |
+
top: calc(100% + 4px) !important;
|
| 1749 |
+
bottom: auto !important;
|
| 1750 |
+
left: 0 !important;
|
| 1751 |
+
right: auto !important;
|
| 1752 |
+
width: 100% !important;
|
| 1753 |
+
max-height: min(280px, 60vh) !important;
|
| 1754 |
+
background: var(--pruna-bg-card) !important;
|
| 1755 |
+
color: var(--pruna-text-primary) !important;
|
| 1756 |
+
border: 1px solid var(--pruna-border) !important;
|
| 1757 |
+
border-radius: 10px !important;
|
| 1758 |
+
overflow: auto !important;
|
| 1759 |
+
z-index: 50 !important;
|
| 1760 |
+
}
|
| 1761 |
+
.options .item:first-child,
|
| 1762 |
+
.options li:first-child {
|
| 1763 |
+
border-top-left-radius: 10px !important;
|
| 1764 |
+
border-top-right-radius: 10px !important;
|
| 1765 |
+
}
|
| 1766 |
+
.options .item:last-child,
|
| 1767 |
+
.options li:last-child {
|
| 1768 |
+
border-bottom-left-radius: 10px !important;
|
| 1769 |
+
border-bottom-right-radius: 10px !important;
|
| 1770 |
+
}
|
| 1771 |
+
.dropdown-arrow-inner {
|
| 1772 |
+
background: var(--pruna-bg-card) !important;
|
| 1773 |
+
color: var(--pruna-text-primary) !important;
|
| 1774 |
+
border-color: var(--pruna-border) !important;
|
| 1775 |
+
}
|
| 1776 |
+
.options .item,
|
| 1777 |
+
.options li {
|
| 1778 |
+
color: var(--pruna-text-primary) !important;
|
| 1779 |
+
}
|
| 1780 |
+
.options .item:hover,
|
| 1781 |
+
.options li:hover,
|
| 1782 |
+
.options .item.selected {
|
| 1783 |
+
background: var(--pruna-dropdown-hover) !important;
|
| 1784 |
+
color: var(--pruna-text-primary) !important;
|
| 1785 |
+
}
|
| 1786 |
+
input[type="range"] {
|
| 1787 |
+
accent-color: var(--pruna-accent);
|
| 1788 |
}
|
| 1789 |
"""
|
| 1790 |
|
| 1791 |
+
theme = gr.themes.Base(
|
| 1792 |
+
primary_hue=gr.themes.Color(
|
| 1793 |
+
c50="#faf5ff",
|
| 1794 |
+
c100="#f3e8ff",
|
| 1795 |
+
c200="#e9d5ff",
|
| 1796 |
+
c300="#d8b4fe",
|
| 1797 |
+
c400="#c084fc",
|
| 1798 |
+
c500="#9334e9",
|
| 1799 |
+
c600="#9334e9",
|
| 1800 |
+
c700="#7e22ce",
|
| 1801 |
+
c800="#6b21a8",
|
| 1802 |
+
c900="#3b0764",
|
| 1803 |
+
c950="#120b1b",
|
| 1804 |
+
),
|
| 1805 |
+
secondary_hue="zinc",
|
| 1806 |
+
neutral_hue="zinc",
|
| 1807 |
+
font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
|
| 1808 |
+
font_mono=[gr.themes.GoogleFont("IBM Plex Mono"), "ui-monospace", "monospace"],
|
| 1809 |
+
).set(
|
| 1810 |
+
body_background_fill="#f6f3fa",
|
| 1811 |
+
body_background_fill_dark="#120b1b",
|
| 1812 |
+
body_text_color="#18181b",
|
| 1813 |
+
body_text_color_dark="#fafafa",
|
| 1814 |
+
body_text_color_subdued="#71717a",
|
| 1815 |
+
body_text_color_subdued_dark="#a3a3a3",
|
| 1816 |
+
background_fill_primary="#f6f3fa",
|
| 1817 |
+
background_fill_primary_dark="#120b1b",
|
| 1818 |
+
background_fill_secondary="#ffffff",
|
| 1819 |
+
background_fill_secondary_dark="#171021",
|
| 1820 |
+
block_background_fill="#ffffff",
|
| 1821 |
+
block_background_fill_dark="#1d1429",
|
| 1822 |
+
block_border_color="#e4d7f2",
|
| 1823 |
+
block_border_color_dark="#4a3962bf",
|
| 1824 |
+
block_label_text_color="#71717a",
|
| 1825 |
+
block_label_text_color_dark="#a3a3a3",
|
| 1826 |
+
block_title_text_color="#18181b",
|
| 1827 |
+
block_title_text_color_dark="#fafafa",
|
| 1828 |
+
border_color_primary="#e4d7f2",
|
| 1829 |
+
border_color_primary_dark="#4a3962bf",
|
| 1830 |
+
button_primary_background_fill="#9334e9",
|
| 1831 |
+
button_primary_background_fill_dark="#9334e9",
|
| 1832 |
+
button_primary_background_fill_hover="#7c22ce",
|
| 1833 |
+
button_primary_background_fill_hover_dark="#8f35ff",
|
| 1834 |
+
button_primary_text_color="#ffffff",
|
| 1835 |
+
button_primary_text_color_dark="#ffffff",
|
| 1836 |
+
button_secondary_background_fill="#ffffff",
|
| 1837 |
+
button_secondary_background_fill_dark="#ffffff05",
|
| 1838 |
+
button_secondary_background_fill_hover="#f3e8ff",
|
| 1839 |
+
button_secondary_background_fill_hover_dark="rgba(147, 52, 233, 0.10)",
|
| 1840 |
+
button_secondary_text_color="#18181b",
|
| 1841 |
+
button_secondary_text_color_dark="#fafafa",
|
| 1842 |
+
button_secondary_border_color="#e4d7f2",
|
| 1843 |
+
button_secondary_border_color_dark="#d8b4fe26",
|
| 1844 |
+
input_background_fill="#ffffff",
|
| 1845 |
+
input_background_fill_dark="#171021",
|
| 1846 |
+
input_border_color="#e4d7f2",
|
| 1847 |
+
input_border_color_dark="#4a3962bf",
|
| 1848 |
+
input_placeholder_color="#71717a",
|
| 1849 |
+
input_placeholder_color_dark="#a3a3a3",
|
| 1850 |
+
link_text_color="#6d28d9",
|
| 1851 |
+
link_text_color_dark="#bf8bff",
|
| 1852 |
+
link_text_color_hover="#7c22ce",
|
| 1853 |
+
link_text_color_hover_dark="#d2a8ff",
|
| 1854 |
+
shadow_drop="none",
|
| 1855 |
+
shadow_drop_lg="none",
|
| 1856 |
)
|
| 1857 |
|
| 1858 |
+
|
| 1859 |
space_root = Path(__file__).resolve().parent
|
| 1860 |
data_dir = space_root / "data"
|
| 1861 |
|
|
|
|
| 1914 |
}
|
| 1915 |
|
| 1916 |
|
| 1917 |
+
def _as_numeric(df, columns):
|
| 1918 |
+
for column in columns:
|
| 1919 |
+
if column in df.columns:
|
| 1920 |
+
df[column] = pd.to_numeric(df[column], errors="coerce")
|
| 1921 |
+
return df
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1922 |
|
| 1923 |
|
| 1924 |
def load_oneig_dataframe(path):
|
|
|
|
| 1926 |
df = df.rename(
|
| 1927 |
columns={
|
| 1928 |
"Owner": "Endpoint Owner",
|
|
|
|
| 1929 |
"Anime Alignment Score": "OneIG (Anime Alignment)",
|
| 1930 |
"Human Alignment Score": "OneIG (Human Alignment)",
|
| 1931 |
"Object Alignment Score": "OneIG (Object Alignment)",
|
|
|
|
| 1934 |
"OneIG (General Object) (Alignment Score)": "OneIG (Object Alignment)",
|
| 1935 |
}
|
| 1936 |
)
|
| 1937 |
+
df = df.drop(
|
| 1938 |
+
columns=[
|
| 1939 |
+
column
|
| 1940 |
+
for column in ("URL", "Device", "Optimization", "Optimization Details")
|
| 1941 |
+
if column in df.columns
|
| 1942 |
+
]
|
| 1943 |
+
)
|
| 1944 |
if "Optimized" in df.columns:
|
| 1945 |
df["Optimized"] = df["Optimized"].map(
|
| 1946 |
{True: "Yes", False: "No", "TRUE": "Yes", "FALSE": "No"}
|
| 1947 |
).fillna(df["Optimized"])
|
| 1948 |
|
| 1949 |
+
return _as_numeric(
|
| 1950 |
+
df,
|
| 1951 |
+
[
|
| 1952 |
+
"Price / Image (USD)",
|
| 1953 |
+
"Median Generation Time (s)",
|
| 1954 |
+
"Min Generation Time (s)",
|
| 1955 |
+
"OneIG (Anime Alignment)",
|
| 1956 |
+
"OneIG (Human Alignment)",
|
| 1957 |
+
"OneIG (Object Alignment)",
|
| 1958 |
+
"OneIG Anime Elo",
|
| 1959 |
+
"OneIG Human Elo",
|
| 1960 |
+
"OneIG Object Elo",
|
| 1961 |
+
"P-Judge Overall",
|
| 1962 |
+
"Rapidata Elo",
|
| 1963 |
+
],
|
| 1964 |
+
)
|
| 1965 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1966 |
|
| 1967 |
+
def load_artificial_analysis_dataframe(path):
|
| 1968 |
+
"""Load the Artificial Analysis text-to-image leaderboard."""
|
| 1969 |
+
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
|
| 1970 |
+
df = df.rename(
|
| 1971 |
+
columns={
|
| 1972 |
+
"model": "Model",
|
| 1973 |
+
"elo": "Artificial Analysis Elo",
|
| 1974 |
+
"price_per_image_usd": "Price / Image (USD)",
|
| 1975 |
+
}
|
| 1976 |
+
)
|
| 1977 |
+
df["Model"] = df["Model"].astype(str).str.strip()
|
| 1978 |
+
return _as_numeric(
|
| 1979 |
+
df, ["Artificial Analysis Elo", "Price / Image (USD)"]
|
| 1980 |
+
).reset_index(drop=True)
|
| 1981 |
|
| 1982 |
+
|
| 1983 |
+
ARENA_CATEGORY_COLUMNS = {
|
| 1984 |
+
"product_branding_commercial_design_elo": "Arena Branding / Commercial Elo",
|
| 1985 |
+
"3d_imaging_and_modeling_elo": "Arena 3D Imaging Elo",
|
| 1986 |
+
"cartoon_anime_and_fantasy_elo": "Arena Cartoon / Anime Elo",
|
| 1987 |
+
"photorealistic_and_cinematic_imagery_elo": "Arena Photorealistic Elo",
|
| 1988 |
+
"art_elo": "Arena Art Elo",
|
| 1989 |
+
"portraits_elo": "Arena Portraits Elo",
|
| 1990 |
+
"text_rendering_elo": "Arena Text Rendering Elo",
|
| 1991 |
+
}
|
| 1992 |
+
|
| 1993 |
+
|
| 1994 |
+
def load_arena_ai_dataframe(path):
|
| 1995 |
+
"""Load the Arena AI text-to-image leaderboard (overall + category Elos)."""
|
| 1996 |
+
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
|
| 1997 |
+
rename_map = {"model": "Model", "elo": "Arena Elo"}
|
| 1998 |
+
rename_map.update(
|
| 1999 |
+
{
|
| 2000 |
+
source: label
|
| 2001 |
+
for source, label in ARENA_CATEGORY_COLUMNS.items()
|
| 2002 |
+
if source in df.columns
|
| 2003 |
+
}
|
| 2004 |
+
)
|
| 2005 |
+
df = df.rename(columns=rename_map)
|
| 2006 |
+
df["Model"] = df["Model"].astype(str).str.strip()
|
| 2007 |
+
score_columns = [
|
| 2008 |
+
column
|
| 2009 |
+
for column in ["Arena Elo", *ARENA_CATEGORY_COLUMNS.values()]
|
| 2010 |
+
if column in df.columns
|
| 2011 |
+
]
|
| 2012 |
+
df = _as_numeric(df, score_columns)
|
| 2013 |
+
ordered = ["Model", *score_columns]
|
| 2014 |
+
return df[[column for column in ordered if column in df.columns]].reset_index(
|
| 2015 |
+
drop=True
|
| 2016 |
+
)
|
| 2017 |
|
| 2018 |
|
| 2019 |
def load_qwen_combined_dataframe(path):
|
|
|
|
| 2026 |
df = df[~df["Model"].astype(str).str.startswith("#")].copy()
|
| 2027 |
df["Model"] = df["Model"].astype(str).str.strip()
|
| 2028 |
|
| 2029 |
+
return _as_numeric(
|
| 2030 |
+
df,
|
| 2031 |
+
[
|
| 2032 |
+
"Price / Image (USD)",
|
| 2033 |
+
"Median Generation Time (s)",
|
| 2034 |
+
"Min Generation Time (s)",
|
| 2035 |
+
"P-Judge Overall",
|
| 2036 |
+
"Rapidata Elo",
|
| 2037 |
+
"Datapoint Elo",
|
| 2038 |
+
],
|
| 2039 |
+
).reset_index(drop=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2040 |
|
| 2041 |
|
| 2042 |
df = load_oneig_dataframe(oneig_path)
|
|
|
|
| 2055 |
oneig_df["OneIG Overall Score"] = oneig_df[oneig_metric_columns].mean(
|
| 2056 |
axis=1, skipna=True
|
| 2057 |
)
|
|
|
|
|
|
|
|
|
|
| 2058 |
|
| 2059 |
oneig_display_columns = [
|
| 2060 |
col
|
|
|
|
| 2063 |
"Endpoint Owner",
|
| 2064 |
"Model",
|
| 2065 |
"Optimized",
|
| 2066 |
+
"OneIG Overall Score",
|
| 2067 |
*oneig_metric_columns,
|
| 2068 |
"OneIG Anime Elo",
|
| 2069 |
"OneIG Human Elo",
|
|
|
|
| 2074 |
"Median Generation Time (s)",
|
| 2075 |
"Min Generation Time (s)",
|
| 2076 |
"Price / Image (USD)",
|
|
|
|
| 2077 |
]
|
| 2078 |
if col in oneig_df.columns
|
| 2079 |
]
|
|
|
|
|
|
|
| 2080 |
|
| 2081 |
oneig_combined_dir = _resolve_data_path(
|
| 2082 |
data_dir / "oneig_combined",
|
|
|
|
| 2090 |
data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 2091 |
space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
|
| 2092 |
)
|
| 2093 |
+
aa_path = _resolve_data_path(
|
| 2094 |
+
data_dir / "artificial_analysis_text_to_image_leaderboard.csv",
|
| 2095 |
+
space_root.parent / "artificial_analysis_text_to_image_leaderboard.csv",
|
| 2096 |
+
)
|
| 2097 |
+
arena_path = _resolve_data_path(
|
| 2098 |
+
data_dir / "arena_ai_text_to_image_leaderboard.csv",
|
| 2099 |
+
space_root.parent / "arena_ai_text_to_image_leaderboard.csv",
|
| 2100 |
+
)
|
| 2101 |
|
| 2102 |
qwen_df = load_qwen_combined_dataframe(qwen_path)
|
| 2103 |
+
aa_df = load_artificial_analysis_dataframe(aa_path)
|
| 2104 |
+
arena_df = load_arena_ai_dataframe(arena_path)
|
| 2105 |
+
qwen_display_columns = [
|
| 2106 |
col
|
| 2107 |
for col in [
|
| 2108 |
+
"Model",
|
| 2109 |
"Datapoint Elo",
|
| 2110 |
"Rapidata Elo",
|
| 2111 |
+
"P-Judge Overall",
|
| 2112 |
+
"Raw Win Rate",
|
| 2113 |
+
"Median Generation Time (s)",
|
| 2114 |
+
"Min Generation Time (s)",
|
| 2115 |
+
"Price / Image (USD)",
|
| 2116 |
]
|
| 2117 |
if col in qwen_df.columns
|
| 2118 |
]
|
| 2119 |
+
aa_display_columns = [
|
| 2120 |
col
|
| 2121 |
for col in [
|
| 2122 |
"Model",
|
| 2123 |
+
"Artificial Analysis Elo",
|
|
|
|
|
|
|
|
|
|
| 2124 |
"Price / Image (USD)",
|
| 2125 |
]
|
| 2126 |
+
if col in aa_df.columns
|
| 2127 |
+
]
|
| 2128 |
+
arena_display_columns = [
|
| 2129 |
+
col
|
| 2130 |
+
for col in [
|
| 2131 |
+
"Model",
|
| 2132 |
+
"Arena Elo",
|
| 2133 |
+
*ARENA_CATEGORY_COLUMNS.values(),
|
| 2134 |
+
]
|
| 2135 |
+
if col in arena_df.columns
|
| 2136 |
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2137 |
|
| 2138 |
oneig_samples = load_sample_comparison_data(oneig_combined_dir)
|
| 2139 |
qwen_samples = load_sample_comparison_data(qwen_combined_dir)
|
| 2140 |
|
| 2141 |
+
metrics = [
|
| 2142 |
+
{"id": "datapoint_elo", "column": "Datapoint Elo"},
|
| 2143 |
+
{"id": "rapidata_elo", "column": "Rapidata Elo"},
|
| 2144 |
+
{"id": "pjudger", "column": "P-Judge Overall"},
|
| 2145 |
+
{"id": "alignment_overall", "column": "OneIG Overall Score"},
|
| 2146 |
+
{"id": "datapoint_elo_anime", "column": "OneIG Anime Elo"},
|
| 2147 |
+
{"id": "datapoint_elo_human", "column": "OneIG Human Elo"},
|
| 2148 |
+
{"id": "datapoint_elo_object", "column": "OneIG Object Elo"},
|
| 2149 |
+
{"id": "aa_elo", "column": "Artificial Analysis Elo"},
|
| 2150 |
+
{"id": "arena_elo", "column": "Arena Elo"},
|
| 2151 |
+
{"id": "arena_branding", "column": "Arena Branding / Commercial Elo"},
|
| 2152 |
+
{"id": "arena_3d", "column": "Arena 3D Imaging Elo"},
|
| 2153 |
+
{"id": "arena_cartoon", "column": "Arena Cartoon / Anime Elo"},
|
| 2154 |
+
{"id": "arena_photo", "column": "Arena Photorealistic Elo"},
|
| 2155 |
+
{"id": "arena_art", "column": "Arena Art Elo"},
|
| 2156 |
+
{"id": "arena_portraits", "column": "Arena Portraits Elo"},
|
| 2157 |
+
{"id": "arena_text", "column": "Arena Text Rendering Elo"},
|
| 2158 |
+
]
|
| 2159 |
+
|
| 2160 |
+
|
| 2161 |
+
def _metric_ids_for(data, metric_ids):
|
| 2162 |
+
columns = getattr(data, "columns", [])
|
| 2163 |
+
return [
|
| 2164 |
+
metric_id
|
| 2165 |
+
for metric_id in metric_ids
|
| 2166 |
+
if any(metric["id"] == metric_id and metric["column"] in columns for metric in metrics)
|
| 2167 |
+
]
|
| 2168 |
+
|
| 2169 |
+
|
| 2170 |
+
qwen_metric_ids = _metric_ids_for(
|
| 2171 |
+
qwen_df, ["datapoint_elo", "rapidata_elo", "pjudger"]
|
| 2172 |
+
)
|
| 2173 |
+
oneig_metric_ids = _metric_ids_for(
|
| 2174 |
+
oneig_df,
|
| 2175 |
+
[
|
| 2176 |
+
"alignment_overall",
|
| 2177 |
+
"rapidata_elo",
|
| 2178 |
+
"pjudger",
|
| 2179 |
+
"datapoint_elo_anime",
|
| 2180 |
+
"datapoint_elo_human",
|
| 2181 |
+
"datapoint_elo_object",
|
| 2182 |
+
],
|
| 2183 |
+
)
|
| 2184 |
+
aa_metric_ids = _metric_ids_for(aa_df, ["aa_elo"])
|
| 2185 |
+
arena_metric_ids = _metric_ids_for(
|
| 2186 |
+
arena_df,
|
| 2187 |
+
[
|
| 2188 |
+
"arena_elo",
|
| 2189 |
+
"arena_branding",
|
| 2190 |
+
"arena_3d",
|
| 2191 |
+
"arena_cartoon",
|
| 2192 |
+
"arena_photo",
|
| 2193 |
+
"arena_art",
|
| 2194 |
+
"arena_portraits",
|
| 2195 |
+
"arena_text",
|
| 2196 |
+
],
|
| 2197 |
+
)
|
| 2198 |
+
|
| 2199 |
+
datasets = [
|
| 2200 |
+
{
|
| 2201 |
+
"id": "qwen",
|
| 2202 |
+
"name": "Qwen Image Dataset",
|
| 2203 |
+
"data": qwen_df,
|
| 2204 |
+
"columns": qwen_display_columns,
|
| 2205 |
+
"metric_ids": qwen_metric_ids,
|
| 2206 |
+
"note": "Rapidata Elo is a metric on this dataset, not a dataset of its own.",
|
| 2207 |
+
"samples": qwen_samples,
|
| 2208 |
+
},
|
| 2209 |
{
|
| 2210 |
"id": "oneig",
|
| 2211 |
+
"name": "OneIG Alignment Dataset",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2212 |
"data": oneig_df,
|
| 2213 |
"columns": oneig_display_columns,
|
| 2214 |
+
"metric_ids": oneig_metric_ids,
|
|
|
|
|
|
|
|
|
|
| 2215 |
"note": (
|
| 2216 |
+
"Alignment Overall is the mean of the available category scores. "
|
| 2217 |
+
"Missing categories are skipped for that model."
|
| 2218 |
),
|
| 2219 |
"samples": oneig_samples,
|
| 2220 |
},
|
| 2221 |
{
|
| 2222 |
+
"id": "artificial_analysis",
|
| 2223 |
+
"name": "Artificial Analysis Dataset",
|
| 2224 |
+
"data": aa_df,
|
| 2225 |
+
"columns": aa_display_columns,
|
| 2226 |
+
"metric_ids": aa_metric_ids,
|
| 2227 |
+
"note": "",
|
| 2228 |
+
"samples": None,
|
| 2229 |
+
},
|
| 2230 |
+
{
|
| 2231 |
+
"id": "arena_ai",
|
| 2232 |
+
"name": "Arena AI Dataset",
|
| 2233 |
+
"data": arena_df,
|
| 2234 |
+
"columns": arena_display_columns,
|
| 2235 |
+
"metric_ids": arena_metric_ids,
|
| 2236 |
+
"note": "",
|
| 2237 |
+
"samples": None,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2238 |
},
|
| 2239 |
]
|
| 2240 |
+
datasets = [dataset for dataset in datasets if dataset["metric_ids"]]
|
| 2241 |
+
|
| 2242 |
+
DEFAULT_DATASET_ID = next(
|
| 2243 |
+
(dataset["id"] for dataset in datasets if dataset["id"] == "qwen"),
|
| 2244 |
+
datasets[0]["id"] if datasets else None,
|
| 2245 |
+
)
|
| 2246 |
+
DEFAULT_METRIC_ID = None
|
| 2247 |
|
| 2248 |
|
| 2249 |
custom_head = """
|
| 2250 |
<script>
|
| 2251 |
+
(function () {
|
| 2252 |
+
var mode = "dark";
|
| 2253 |
+
try {
|
| 2254 |
+
var stored = localStorage.getItem("inferbench-theme");
|
| 2255 |
+
if (stored === "light" || stored === "dark") mode = stored;
|
| 2256 |
+
else {
|
| 2257 |
+
var param = new URLSearchParams(location.search).get("__theme");
|
| 2258 |
+
if (param === "light" || param === "dark") mode = param;
|
| 2259 |
+
}
|
| 2260 |
+
} catch (e) {}
|
| 2261 |
+
document.documentElement.classList.toggle("dark", mode === "dark");
|
| 2262 |
+
document.documentElement.dataset.theme = mode;
|
| 2263 |
+
})();
|
| 2264 |
+
</script>
|
| 2265 |
+
<link rel="preconnect" href="https://fonts.googleapis.com" />
|
| 2266 |
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
|
| 2267 |
+
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet" />
|
| 2268 |
+
<script>
|
| 2269 |
+
(() => {
|
| 2270 |
+
const STORAGE_KEY = "inferbench-theme";
|
| 2271 |
+
const PLOT_LAYOUT = {
|
| 2272 |
+
dark: {
|
| 2273 |
+
paper_bgcolor: "#171021",
|
| 2274 |
+
plot_bgcolor: "#1d1429",
|
| 2275 |
+
"font.color": "#d4d4d4",
|
| 2276 |
+
"legend.font.color": "#d4d4d4",
|
| 2277 |
+
"xaxis.gridcolor": "rgba(74, 57, 98, 0.55)",
|
| 2278 |
+
"xaxis.color": "#d4d4d4",
|
| 2279 |
+
"xaxis.title.font.color": "#fafafa",
|
| 2280 |
+
"xaxis.tickfont.color": "#a3a3a3",
|
| 2281 |
+
"yaxis.gridcolor": "rgba(74, 57, 98, 0.55)",
|
| 2282 |
+
"yaxis.color": "#d4d4d4",
|
| 2283 |
+
"yaxis.title.font.color": "#fafafa",
|
| 2284 |
+
"yaxis.tickfont.color": "#a3a3a3",
|
| 2285 |
+
},
|
| 2286 |
+
light: {
|
| 2287 |
+
paper_bgcolor: "#f6f3fa",
|
| 2288 |
+
plot_bgcolor: "#ffffff",
|
| 2289 |
+
"font.color": "#3f3f46",
|
| 2290 |
+
"legend.font.color": "#3f3f46",
|
| 2291 |
+
"xaxis.gridcolor": "rgba(126, 34, 206, 0.12)",
|
| 2292 |
+
"xaxis.color": "#3f3f46",
|
| 2293 |
+
"xaxis.title.font.color": "#18181b",
|
| 2294 |
+
"xaxis.tickfont.color": "#71717a",
|
| 2295 |
+
"yaxis.gridcolor": "rgba(126, 34, 206, 0.12)",
|
| 2296 |
+
"yaxis.color": "#3f3f46",
|
| 2297 |
+
"yaxis.title.font.color": "#18181b",
|
| 2298 |
+
"yaxis.tickfont.color": "#71717a",
|
| 2299 |
+
},
|
| 2300 |
+
};
|
| 2301 |
+
|
| 2302 |
+
const currentMode = () => {
|
| 2303 |
+
try {
|
| 2304 |
+
const stored = localStorage.getItem(STORAGE_KEY);
|
| 2305 |
+
if (stored === "light" || stored === "dark") return stored;
|
| 2306 |
+
} catch (e) {}
|
| 2307 |
+
const param = new URLSearchParams(location.search).get("__theme");
|
| 2308 |
+
if (param === "light" || param === "dark") return param;
|
| 2309 |
+
return "dark";
|
| 2310 |
+
};
|
| 2311 |
+
|
| 2312 |
+
const queryAll = (selector) => {
|
| 2313 |
+
const found = [...document.querySelectorAll(selector)];
|
| 2314 |
+
document.querySelectorAll("gradio-app").forEach((app) => {
|
| 2315 |
+
if (app.shadowRoot) found.push(...app.shadowRoot.querySelectorAll(selector));
|
| 2316 |
+
});
|
| 2317 |
+
return found;
|
| 2318 |
+
};
|
| 2319 |
+
|
| 2320 |
+
const restylePlots = (mode) => {
|
| 2321 |
+
const layout = PLOT_LAYOUT[mode];
|
| 2322 |
+
if (!layout || typeof Plotly === "undefined") return;
|
| 2323 |
+
queryAll(".js-plotly-plot").forEach((gd) => {
|
| 2324 |
+
try { Plotly.relayout(gd, layout); } catch (e) {}
|
| 2325 |
+
});
|
| 2326 |
+
};
|
| 2327 |
+
|
| 2328 |
+
const applyMode = (mode, persist) => {
|
| 2329 |
+
const dark = mode === "dark";
|
| 2330 |
+
const targets = [
|
| 2331 |
+
document.documentElement,
|
| 2332 |
+
document.body,
|
| 2333 |
+
...queryAll(".gradio-container"),
|
| 2334 |
+
...document.querySelectorAll("gradio-app"),
|
| 2335 |
+
].filter(Boolean);
|
| 2336 |
+
targets.forEach((el) => {
|
| 2337 |
+
el.classList.toggle("dark", dark);
|
| 2338 |
+
el.classList.toggle("light", !dark);
|
| 2339 |
+
});
|
| 2340 |
+
document.documentElement.dataset.theme = mode;
|
| 2341 |
+
const label = dark ? "Switch to light mode" : "Switch to dark mode";
|
| 2342 |
+
queryAll(".theme-toggle").forEach((btn) => {
|
| 2343 |
+
btn.setAttribute("data-mode", mode);
|
| 2344 |
+
btn.setAttribute("aria-label", label);
|
| 2345 |
+
btn.setAttribute("title", label);
|
| 2346 |
+
});
|
| 2347 |
+
if (persist) {
|
| 2348 |
+
try { localStorage.setItem(STORAGE_KEY, mode); } catch (e) {}
|
| 2349 |
+
try {
|
| 2350 |
+
const url = new URL(location.href);
|
| 2351 |
+
url.searchParams.set("__theme", mode);
|
| 2352 |
+
history.replaceState(null, "", url);
|
| 2353 |
+
} catch (e) {}
|
| 2354 |
+
}
|
| 2355 |
+
restylePlots(mode);
|
| 2356 |
+
};
|
| 2357 |
+
|
| 2358 |
+
applyMode(currentMode(), false);
|
| 2359 |
+
|
| 2360 |
+
document.addEventListener("click", (event) => {
|
| 2361 |
+
const path = typeof event.composedPath === "function" ? event.composedPath() : [event.target];
|
| 2362 |
+
const btn = path.find((el) => el.classList && el.classList.contains("theme-toggle"));
|
| 2363 |
+
if (!btn) return;
|
| 2364 |
+
event.preventDefault();
|
| 2365 |
+
const next = currentMode() === "dark" ? "light" : "dark";
|
| 2366 |
+
applyMode(next, true);
|
| 2367 |
+
});
|
| 2368 |
+
|
| 2369 |
+
const sync = () => applyMode(currentMode(), false);
|
| 2370 |
+
document.addEventListener("DOMContentLoaded", sync);
|
| 2371 |
+
window.addEventListener("load", () => {
|
| 2372 |
+
sync();
|
| 2373 |
+
setTimeout(sync, 80);
|
| 2374 |
+
setTimeout(sync, 400);
|
| 2375 |
+
});
|
| 2376 |
+
|
| 2377 |
+
const startObserver = () => {
|
| 2378 |
+
if (!document.body || window.__inferbenchThemeObserver) return;
|
| 2379 |
+
window.__inferbenchThemeObserver = true;
|
| 2380 |
+
const onMutations = (mutations) => {
|
| 2381 |
+
const mode = currentMode();
|
| 2382 |
+
const isDark = document.documentElement.classList.contains("dark");
|
| 2383 |
+
const added = mutations.flatMap((mutation) => [...mutation.addedNodes]).filter((node) => node.nodeType === 1);
|
| 2384 |
+
const hasContainer = added.some((node) => (
|
| 2385 |
+
node.classList?.contains("gradio-container") ||
|
| 2386 |
+
node.querySelector?.(".gradio-container, .theme-toggle")
|
| 2387 |
+
));
|
| 2388 |
+
if ((mode === "dark") !== isDark || hasContainer) applyMode(mode, false);
|
| 2389 |
+
const hasPlot = added.some((node) => (
|
| 2390 |
+
node.classList?.contains("js-plotly-plot") ||
|
| 2391 |
+
node.querySelector?.(".js-plotly-plot")
|
| 2392 |
+
));
|
| 2393 |
+
if (hasPlot) restylePlots(mode);
|
| 2394 |
+
};
|
| 2395 |
+
new MutationObserver(onMutations).observe(document.body, { childList: true, subtree: true });
|
| 2396 |
+
document.querySelectorAll("gradio-app").forEach((app) => {
|
| 2397 |
+
if (app.shadowRoot) {
|
| 2398 |
+
new MutationObserver(onMutations).observe(app.shadowRoot, { childList: true, subtree: true });
|
| 2399 |
+
}
|
| 2400 |
+
});
|
| 2401 |
+
};
|
| 2402 |
+
if (document.body) startObserver();
|
| 2403 |
+
else document.addEventListener("DOMContentLoaded", startObserver);
|
| 2404 |
+
})();
|
| 2405 |
+
</script>
|
| 2406 |
+
<script>
|
| 2407 |
(() => {
|
| 2408 |
if (window.__inferbenchLeaderboardSortBound) return;
|
| 2409 |
window.__inferbenchLeaderboardSortBound = true;
|
|
|
|
| 2458 |
});
|
| 2459 |
})();
|
| 2460 |
</script>
|
| 2461 |
+
<script>
|
| 2462 |
+
(() => {
|
| 2463 |
+
if (window.__inferbenchChipScrollBound) return;
|
| 2464 |
+
window.__inferbenchChipScrollBound = true;
|
| 2465 |
+
document.addEventListener("wheel", (event) => {
|
| 2466 |
+
const row = event.target.closest?.(".view-filters .filter-chips .wrap-inner");
|
| 2467 |
+
if (!row || row.scrollWidth <= row.clientWidth + 1) return;
|
| 2468 |
+
if (Math.abs(event.deltaY) < Math.abs(event.deltaX)) return;
|
| 2469 |
+
row.scrollLeft += event.deltaY;
|
| 2470 |
+
event.preventDefault();
|
| 2471 |
+
}, { capture: true, passive: false });
|
| 2472 |
+
})();
|
| 2473 |
+
</script>
|
| 2474 |
"""
|
| 2475 |
|
| 2476 |
with gr.Blocks(
|
| 2477 |
+
title="P-Bench",
|
| 2478 |
+
theme=theme,
|
| 2479 |
+
fill_width=True,
|
| 2480 |
+
css=custom_css,
|
| 2481 |
+
head=custom_head,
|
| 2482 |
) as demo:
|
| 2483 |
render_header()
|
| 2484 |
+
render_image_workspace(
|
| 2485 |
+
datasets,
|
| 2486 |
+
metrics,
|
| 2487 |
+
DEFAULT_DATASET_ID,
|
| 2488 |
+
DEFAULT_METRIC_ID,
|
| 2489 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2490 |
render_footer()
|
| 2491 |
|
| 2492 |
|
| 2493 |
if __name__ == "__main__":
|
| 2494 |
+
demo.launch(
|
| 2495 |
+
ssr_mode=False,
|
| 2496 |
+
allowed_paths=[str(space_root)],
|
| 2497 |
+
)
|
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model,elo,product_branding_commercial_design_elo,3d_imaging_and_modeling_elo,cartoon_anime_and_fantasy_elo,photorealistic_and_cinematic_imagery_elo,art_elo,portraits_elo,text_rendering_elo
|
| 2 |
+
gpt-image-2 (medium),1381,1391,1361,1397,1380,1370,1429,1425
|
| 3 |
+
mai-image-2.6-preview,1336,1344,1362,1355,1325,1345,1365,1374
|
| 4 |
+
grok-imagine-image-2.0 (low),1316,1309,1268,1321,1327,1306,1371,1353
|
| 5 |
+
reve-2.1,1302,1314,1274,1310,1302,1289,1317,1341
|
| 6 |
+
muse-image,1282,1269,1255,1289,1293,1275,1313,1304
|
| 7 |
+
reve-2.0,1270,1279,1220,1275,1280,1259,1283,1297
|
| 8 |
+
gemini-3.1-flash-image (nano-banana-2) [web-search],1264,1271,1260,1268,1270,1251,1266,1295
|
| 9 |
+
seedream-5.0-pro,1258,1234,1246,1273,1280,1269,1291,1255
|
| 10 |
+
qwen-image-3.0-pro,1257,1258,1240,1271,1257,1257,1273,1289
|
| 11 |
+
mai-image-2.5,1256,1261,1248,1270,1252,1278,1259,1284
|
| 12 |
+
gemini-3.1-flash-lite-image (nano-banana-2-lite),1251,1250,1225,1258,1260,1257,1248,1275
|
| 13 |
+
gemini-3-pro-image-2k (nano-banana-pro),1246,1246,1231,1242,1260,1225,1258,1270
|
| 14 |
+
gpt-image-1.5-high-fidelity,1239,1241,1216,1243,1249,1224,1261,1254
|
| 15 |
+
gemini-3-pro-image-preview (nano-banana-pro),1232,1237,1209,1238,1243,1216,1236,1254
|
| 16 |
+
ideogram-4.0-quality,1204,1224,1180,1195,1203,1199,1201,1238
|
| 17 |
+
qwen-image-2.0-pro-2026-06-22,1191,1187,1198,1191,1205,1177,1216,1201
|
| 18 |
+
uni-1.1-max,1188,1197,1172,1194,1188,1195,1158,1217
|
| 19 |
+
mai-image-2,1183,1175,1179,1180,1203,1177,1207,1175
|
| 20 |
+
uni-1.1,1181,1181,1167,1189,1180,1188,1176,1202
|
| 21 |
+
Cosmos3-Super-Text2Image (Agentic),1175,1162,1170,1191,1189,1195,1212,1151
|
| 22 |
+
grok-imagine-image,1171,1175,1176,1177,1179,1179,1187,1185
|
| 23 |
+
recraft-v4.1-utility-pro,1169,1167,1147,1177,1180,1178,1169,1162
|
| 24 |
+
flux-2-max,1162,1165,1161,1169,1161,1170,1157,1167
|
| 25 |
+
grok-imagine-image-pro,1161,1168,1165,1163,1169,1165,1172,1182
|
| 26 |
+
flux-2-flex,1157,1173,1172,1162,1147,1156,1145,1172
|
| 27 |
+
flux-2-pro,1155,1157,1150,1161,1152,1163,1146,1157
|
| 28 |
+
Cosmos3-Super-Text2Image,1155,1129,1166,1177,1155,1173,1176,1122
|
| 29 |
+
reve-v1.5,1154,1182,1143,1141,1147,1144,1149,1185
|
| 30 |
+
hunyuan-image-3.0,1151,1153,1168,1163,1152,1153,1146,1150
|
| 31 |
+
gemini-2.5-flash-image-preview (nano-banana),1150,1152,1142,1145,1157,1145,1168,1154
|
| 32 |
+
imagen-ultra-4.0-generate-001,1148,1151,1141,1141,1156,1128,1168,1148
|
| 33 |
+
seedream-4.5,1147,1140,1147,1149,1166,1154,1166,1141
|
| 34 |
+
flux-2-dev,1145,1155,1149,1144,1143,1150,1133,1159
|
| 35 |
+
seedream-4-2k,1140,1134,1150,1149,1151,1164,1142,1143
|
| 36 |
+
seedream-5.0-lite,1137,1129,1138,1140,1154,1137,1150,1134
|
| 37 |
+
wan2.6-t2i,1136,1148,1146,1146,1128,1156,1123,1149
|
| 38 |
+
recraft-v4.1-pro,1130,1138,1111,1122,1143,1154,1144,1156
|
| 39 |
+
imagen-4.0-generate-001,1129,1129,1123,1120,1136,1110,1145,1123
|
| 40 |
+
qwen-image-2512,1125,1128,1135,1125,1135,1133,1143,1122
|
| 41 |
+
krea-2-medium,1122,1119,1132,1140,1127,1152,1148,1124
|
| 42 |
+
wan2.5-t2i-preview,1117,1138,1135,1120,1102,1120,1094,1142
|
| 43 |
+
hidream-o1-image,1117,1128,1121,1121,1111,1122,1106,1135
|
| 44 |
+
seedream-4-fal,1116,1128,1138,1109,1121,1119,1116,1124
|
| 45 |
+
gpt-image-1,1115,1115,1114,1115,1115,1115,1115,1115
|
| 46 |
+
recraft-v4,1114,1126,1076,1111,1119,1123,1119,1139
|
| 47 |
+
seedream-4-high-res-fal,1113,1113,1121,1109,1129,1112,1133,1110
|
| 48 |
+
krea-2-turbo,1111,1109,1133,1122,1123,1139,1138,1097
|
| 49 |
+
gpt-image-1-mini,1109,1109,1108,1108,1114,1113,1112,1104
|
| 50 |
+
krea-2-large,1107,1104,1113,1112,1121,1116,1117,1111
|
| 51 |
+
wan2.7-image-pro,1103,1114,1080,1080,1123,1079,1127,1118
|
| 52 |
+
wan2.7-image,1100,1110,1082,1075,1117,1075,1125,1114
|
| 53 |
+
mai-image-1,1093,1086,1096,1100,1103,1104,1095,1070
|
| 54 |
+
seedream-3,1082,1070,1094,1081,1102,1092,1102,1052
|
| 55 |
+
z-image-turbo,1082,1060,1066,1067,1115,1074,1140,1058
|
| 56 |
+
flux-1-kontext-max,1074,1082,1079,1079,1083,1081,1085,1071
|
| 57 |
+
flux-2-klein-9b,1070,1070,1078,1064,1090,1079,1092,1057
|
| 58 |
+
qwen-image-prompt-extend,1061,1087,1082,1062,1047,1055,1051,1082
|
| 59 |
+
flux-1-kontext-pro,1059,1068,1062,1058,1066,1067,1065,1053
|
| 60 |
+
imagen-3.0-generate-002,1058,1057,1046,1048,1075,1059,1077,1045
|
| 61 |
+
qwen-image,1057,1068,1064,1052,1062,1055,1076,1047
|
| 62 |
+
ideogram-v3-quality,1049,1082,1029,1027,1051,1039,1052,1070
|
| 63 |
+
photon,1035,1052,1033,1026,1035,1046,1023,1041
|
| 64 |
+
p-image,1034,1031,1031,1014,1074,1041,1100,1021
|
| 65 |
+
flux-2-klein-4b,1030,1026,1030,1023,1047,1028,1054,1015
|
| 66 |
+
runway-gen4,1025,1027,1013,1008,1041,1019,1022,1019
|
| 67 |
+
recraft-v3,1021,1027,1011,978,1054,991,1040,1005
|
| 68 |
+
flux-1.1-pro,1016,1012,1051,1016,1044,1044,1043,995
|
| 69 |
+
lucid-origin,1013,1020,1015,1007,1035,1031,1043,1005
|
| 70 |
+
ideogram-v2,1013,1054,990,995,1019,1017,1019,1046
|
| 71 |
+
glm-image,1010,1036,1018,1003,1000,1013,1000,1033
|
| 72 |
+
gemini-2.0-flash-preview-image-generation,975,1019,976,954,963,965,961,1007
|
| 73 |
+
flux-1-dev-fp8,969,991,978,957,990,978,999,963
|
| 74 |
+
dall-e-3,968,987,987,988,940,1010,924,969
|
| 75 |
+
flux-1-kontext-dev,940,966,943,924,935,927,944,953
|
| 76 |
+
stable-diffusion-v35-large,938,937,964,946,928,998,941,921
|
| 77 |
+
bagel,898,887,866,885,923,911,926,875
|
|
@@ -0,0 +1,146 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model,elo,price_per_image_usd
|
| 2 |
+
GPT Image 2 (high),1370,0.211
|
| 3 |
+
Reve 2.1,1324,0.2
|
| 4 |
+
Nano Banana 2 (Gemini 3.1 Flash Image Preview),1322,0.067
|
| 5 |
+
GPT Image 1.5 (high),1312,0.133
|
| 6 |
+
MAI-Image-2.5,1306,0.0481
|
| 7 |
+
Nano Banana Pro (Gemini 3 Pro Image),1298,0.134
|
| 8 |
+
Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image),1292,0.0336
|
| 9 |
+
Seedream 5.0 Pro,1281,0.09
|
| 10 |
+
grok-imagine-image-quality,1236,0.05
|
| 11 |
+
Qwen Image 2.0 Pro (2026-04-22),1235,0.075
|
| 12 |
+
FLUX.2 [max],1231,0.07
|
| 13 |
+
MAI-Image-2.5-Flash,1230,0.02
|
| 14 |
+
HiDream-O1-Image-1.5,1227,0.08
|
| 15 |
+
Seedream 4.0,1224,0.03
|
| 16 |
+
Luma UNI 1 Max,1223,0.1
|
| 17 |
+
FLUX.2 [flex],1222,0.06
|
| 18 |
+
Krea 2 Medium Turbo,1222,0.015
|
| 19 |
+
Krea 2 Large,1220,0.06
|
| 20 |
+
grok-imagine-image,1218,0.02
|
| 21 |
+
Recraft V4.1 Utility,1217,0.035
|
| 22 |
+
Ideogram 4.0,1217,0.06
|
| 23 |
+
Recraft V4.1 Utility Pro,1217,0.21
|
| 24 |
+
Ideogram 4.0 (Quality),1214,0.1
|
| 25 |
+
Wan2.6 Text to Image,1213,0.03
|
| 26 |
+
Krea 2 Medium,1212,0.03
|
| 27 |
+
MAI-Image-2,1210,0.035
|
| 28 |
+
FLUX.2 [pro],1208,0.03
|
| 29 |
+
Wan 2.6 Image,1206,0.03
|
| 30 |
+
Seedream 4.5,1205,0.04
|
| 31 |
+
GPT Image 1 (high),1205,0.167
|
| 32 |
+
Luma UNI 1,1205,0.0404
|
| 33 |
+
FLUX.2 [dev] Turbo,1199,0.008
|
| 34 |
+
FLUX.2 [dev],1199,0.012
|
| 35 |
+
Seedream 5.0 Lite,1199,0.035
|
| 36 |
+
Ideogram 4.0 Fast (Quality),1199,0.0175
|
| 37 |
+
P-Image-Ideogram (High),1198,0.015
|
| 38 |
+
Recraft V4 Pro,1195,0.25
|
| 39 |
+
Recraft V4.1,1191,0.035
|
| 40 |
+
Nano Banana (Gemini 2.5 Flash Image),1189,0.039
|
| 41 |
+
Recraft V4.1 Pro,1189,0.21
|
| 42 |
+
Imagen 4 Ultra,1189,0.06
|
| 43 |
+
Ideogram 4.0 Instant,1187,0.0075
|
| 44 |
+
Wan 2.7 Pro,1184,0.064
|
| 45 |
+
Recraft V4,1182,0.04
|
| 46 |
+
MAI-Image-2-Efficient,1182,0.022
|
| 47 |
+
P-Image-Ideogram (Medium),1181,0.01
|
| 48 |
+
FLUX.2 [dev] Flash,1179,0.005
|
| 49 |
+
HiDream-O1-Image,1176,
|
| 50 |
+
Ideogram 4.0 Fast,1176,0.0105
|
| 51 |
+
ImagineArt 2.0,1173,0.03
|
| 52 |
+
Qwen Image Max 2512,1173,0.02
|
| 53 |
+
Wan 2.7,1170,0.026
|
| 54 |
+
ImagineArt 1.5 Preview,1165,0.03
|
| 55 |
+
P-Image-Ideogram (Low),1165,0.0075
|
| 56 |
+
Seedream 3.0,1160,0.03
|
| 57 |
+
Vivago 2.1,1152,0.035
|
| 58 |
+
HunyuanImage 3.0 Instruct (Fal),1151,0.09
|
| 59 |
+
Wan 2.5 Preview,1150,0.021
|
| 60 |
+
FLUX.2 [klein] 9B,1146,0.015
|
| 61 |
+
FLUX.1 Kontext [max],1142,0.08
|
| 62 |
+
Kolors 2.1,1142,0.014
|
| 63 |
+
image-1,1141,0.04
|
| 64 |
+
HunyuanImage 3.0 (Fal),1140,0.1
|
| 65 |
+
Qwen Image 2.0 (2026-03-03),1135,0.035
|
| 66 |
+
Z-Image Turbo,1130,0.005
|
| 67 |
+
Vivago 2.0,1125,
|
| 68 |
+
Imagen 3 (v002),1124,0.04
|
| 69 |
+
Lucid Origin Ultra,1122,0.0867
|
| 70 |
+
Eigen Image,1122,0.025
|
| 71 |
+
Imagen 4 Standard,1120,0.04
|
| 72 |
+
Reve Image (Halfmoon),1116,
|
| 73 |
+
Kling Image 3.0 Omni,1115,0.028
|
| 74 |
+
Vidu Q2,1112,0.03
|
| 75 |
+
P-Image-Ideogram (Very Low),1111,0.003
|
| 76 |
+
Qwen Image Plus 2601,1110,0.03
|
| 77 |
+
GPT Image 1 Mini (medium),1110,0.011
|
| 78 |
+
Lucid Origin Fast,1110,0.0179
|
| 79 |
+
FLUX.1 Kontext [pro],1109,0.04
|
| 80 |
+
Dreamina 3.1,1108,0.03
|
| 81 |
+
FLUX1.1 [pro] Ultra,1104,0.06
|
| 82 |
+
Ideogram 3.0,1101,0.06
|
| 83 |
+
FLUX.2 [klein] Base 9B,1099,0.011
|
| 84 |
+
Imagen 4 Fast,1098,0.02
|
| 85 |
+
P-Image,1093,0.005
|
| 86 |
+
FLUX1.1 [pro],1092,0.04
|
| 87 |
+
Midjourney v7 Alpha,1092,
|
| 88 |
+
Ideogram v2,1084,0.08
|
| 89 |
+
FLUX.1 [pro],1083,0.05
|
| 90 |
+
Qwen Image,1082,0.02
|
| 91 |
+
SRPO,1080,0.026
|
| 92 |
+
HunyuanImage 2.1,1078,0.1
|
| 93 |
+
Midjourney v6,1076,
|
| 94 |
+
Ideogram v2 Turbo,1075,0.05
|
| 95 |
+
FIBO,1075,0.04
|
| 96 |
+
Recraft V3,1075,0.04
|
| 97 |
+
Luma Photon,1074,0.019
|
| 98 |
+
HiDream-O1-Image-Dev,1071,0.005
|
| 99 |
+
Image-01,1070,0.01
|
| 100 |
+
HiDream-I1-Dev,1070,0.024
|
| 101 |
+
GLM-Image,1068,0.05
|
| 102 |
+
Z-Image Base,1064,0.01
|
| 103 |
+
MAI Image 1,1064,
|
| 104 |
+
HiDream-I1-Fast,1063,0.012
|
| 105 |
+
FLUX.2 [klein] 4B,1057,0.014
|
| 106 |
+
Midjourney v6.1,1057,
|
| 107 |
+
Infinity 8B,1056,0.0017
|
| 108 |
+
LongCat Image,1053,0.13
|
| 109 |
+
Stable Diffusion 3 Large,1042,
|
| 110 |
+
FLUX.1 [dev],1041,0.025
|
| 111 |
+
Phoenix 1.0 Ultra,1041,0.0762
|
| 112 |
+
FLUX.1 Krea [dev],1036,0.025
|
| 113 |
+
Stable Diffusion 3.5 Large Turbo,1034,0.04
|
| 114 |
+
Stable Diffusion 3.5 Large,1034,0.065
|
| 115 |
+
Phoenix 0.9 Ultra,1028,0.0762
|
| 116 |
+
Ideogram v2a Turbo,1028,0.025
|
| 117 |
+
Ideogram v2a,1027,0.04
|
| 118 |
+
Playground v3 (beta),1015,0.0105
|
| 119 |
+
Krea 1,1013,
|
| 120 |
+
Phoenix 1.0 Fast,1007,0.0194
|
| 121 |
+
Luma Photon Flash,1004,0.005
|
| 122 |
+
FLUX.1 [schnell],1000,0.003
|
| 123 |
+
Step Image Edit 2,1000,0.003
|
| 124 |
+
Runway Gen-4 Image,992,0.08
|
| 125 |
+
Recraft 20B,984,0.022
|
| 126 |
+
Lumina Image v2,978,0.075
|
| 127 |
+
Playground v2.5,972,0.08
|
| 128 |
+
FLUX.2 [klein] Base 4B,972,0.009
|
| 129 |
+
DALLE 3,968,0.04
|
| 130 |
+
DALLE 3 HD,967,0.08
|
| 131 |
+
Stable Diffusion 3.5 Medium,963,0.02
|
| 132 |
+
Sana Sprint 1.6B,939,0.0015
|
| 133 |
+
Amazon Titan G1 v2 (Standard),923,0.01
|
| 134 |
+
Stable Diffusion 3 Medium,922,0.035
|
| 135 |
+
Stable Diffusion 3 Large Turbo,917,
|
| 136 |
+
Amazon Titan G1 (Standard),914,0.01
|
| 137 |
+
Stable Diffusion 1.6,914,0.009
|
| 138 |
+
OmniGen V2,911,0.15
|
| 139 |
+
SDXL Lightning,909,0.0019
|
| 140 |
+
Bagel,908,0.1
|
| 141 |
+
Bria 3.2,906,0.04
|
| 142 |
+
Stable Diffusion XL 1.0,884,0.009
|
| 143 |
+
Stable Diffusion 2.1,754,0.0039
|
| 144 |
+
DALLE 2,745,0.02
|
| 145 |
+
Janus Pro,720,0.0
|
| 146 |
+
Stable Diffusion 1.5,665,0.044
|
|
@@ -1,31 +0,0 @@
|
|
| 1 |
-
{"Platform": "Replicate", "Owner": "Pruna AI", "Device": "1xH100", "Model": "FLUX Schnell", "Optimization": "speed_mode_juiced", "URL": "https://replicate.com/prunaai/flux-schnell", "PartiPromts (ARNIQA)": 0.5665, "PartiPromts (ClipScore)": 27.4594, "PartiPromts (ClipIQA)": 0.8594, "PartiPromts (Sharpness - Laplacian Variance)": 4385.7579, "Long Text Bench (edit_distance)": 178.5125, "Long Text Bench (text_word_accuracy)": null, "HPS (v2.1)": 0.2106, "DrawBench (ClipScore)": 27.9985, "DrawBench (Image Reward)": 1.0057, "GenAI-Bench (VQA)": 0.79, "OneIG (Anime and Stylization) (Alignment Score)": null, "Median Inference Time": 0.9082, "Price per Image": 0.055}
|
| 2 |
-
{"Platform": "fal.ai", "Owner": "fal.ai", "Device": "Undisclosed", "Model": "FLUX 1.1 Pro", "Optimization": "Undisclosed", "URL": "https://fal.ai/models/fal-ai/flux-pro/v1.1", "GenAI-Bench (VQA)": 0.7745, "HPS (v2.1)": 0.2093, "PartiPromts (ARNIQA)": 0.5998, "PartiPromts (ClipScore)": 26.7942, "PartiPromts (ClipIQA)": 0.9282, "PartiPromts (Sharpness - Laplacian Variance)": 14290.615, "Long Text Bench (edit_distance)": 163.5862, "Long Text Bench (text_word_accuracy)": null, "OneIG (Anime and Stylization) (Alignment Score)": null, "DrawBench (ClipScore)": 27.2099, "DrawBench (Image Reward)": 0.8609, "Median Inference Time": 4.031, "Price per Image": 0.04}
|
| 3 |
-
{"Platform": "fal.ai", "Owner": "fal.ai", "Device": "Undisclosed", "Model": "FLUX Dev", "Optimization": "dev", "URL": "https://fal.ai/models/fal-ai/flux/dev", "Long Text Bench (edit_distance)": 177.7438, "Long Text Bench (text_word_accuracy)": null, "OneIG (Anime and Stylization) (Alignment Score)": null, "GenAI-Bench (VQA)": 0.7443, "DrawBench (ClipScore)": 27.0164, "DrawBench (Image Reward)": 0.8933, "PartiPromts (ARNIQA)": 0.6305, "PartiPromts (ClipScore)": 27.2592, "PartiPromts (ClipIQA)": 0.8593, "PartiPromts (Sharpness - Laplacian Variance)": 5212.9337, "HPS (v2.1)": 0.1932, "Median Inference Time": 1.9838, "Price per Image": 0.025}
|
| 4 |
-
{"Platform": "Prodia", "Owner": "Prodia", "Device": "Undisclosed", "Model": "FLUX Dev", "Optimization": "fast", "URL": "https://app.prodia.com/models", "PartiPromts (ARNIQA)": 0.6224, "PartiPromts (ClipScore)": 26.8405, "PartiPromts (ClipIQA)": 0.9159, "PartiPromts (Sharpness - Laplacian Variance)": 5774.8605, "DrawBench (ClipScore)": 26.6909, "DrawBench (Image Reward)": 0.9675, "GenAI-Bench (VQA)": 0.7363, "OneIG (Anime and Stylization) (Alignment Score)": null, "Long Text Bench (edit_distance)": 178.3375, "Long Text Bench (text_word_accuracy)": null, "HPS (v2.1)": 0.1995, "Median Inference Time": 1.9977, "Price per Image": 0.02}
|
| 5 |
-
{"Platform": "Prodia", "Owner": "Prodia", "Device": "Undisclosed", "Model": "FLUX 1.1 Pro", "Optimization": "Undisclosed", "URL": "https://app.prodia.com/models", "HPS (v2.1)": 0.2075, "Long Text Bench (edit_distance)": 178.5125, "Long Text Bench (text_word_accuracy)": null, "DrawBench (ClipScore)": 27.6498, "DrawBench (Image Reward)": 0.9484, "GenAI-Bench (VQA)": 0.7739, "OneIG (Anime and Stylization) (Alignment Score)": null, "PartiPromts (ARNIQA)": 0.6181, "PartiPromts (ClipScore)": 27.3402, "PartiPromts (ClipIQA)": 0.895, "PartiPromts (Sharpness - Laplacian Variance)": 6932.3266, "Median Inference Time": 3.285, "Price per Image": 0.04}
|
| 6 |
-
{"Platform": "fal.ai", "Owner": "fal.ai", "Device": "Undisclosed", "Model": "FLUX Schnell", "Optimization": "schnell", "URL": "https://fal.ai/models/fal-ai/flux/schnell", "HPS (v2.1)": 0.206, "Long Text Bench (edit_distance)": 178.3125, "Long Text Bench (text_word_accuracy)": 0.0003, "PartiPromts (ARNIQA)": 0.6637, "PartiPromts (ClipScore)": 27.755, "PartiPromts (ClipIQA)": 0.8991, "PartiPromts (Sharpness - Laplacian Variance)": 6420.4761, "OneIG (Anime and Stylization) (Alignment Score)": null, "GenAI-Bench (VQA)": 0.7919, "DrawBench (ClipScore)": 28.0585, "DrawBench (Image Reward)": 0.9376, "Median Inference Time": 0.7981, "Price per Image": 0.003}
|
| 7 |
-
{"Platform": "Together AI", "Owner": "Black Forest Labs", "Device": "Undisclosed", "Model": "FLUX Dev", "Optimization": "dev", "URL": "https://www.together.ai/models/flux-1-dev", "Long Text Bench (edit_distance)": 178.0437, "Long Text Bench (text_word_accuracy)": null, "GenAI-Bench (VQA)": 0.7592, "OneIG (Anime and Stylization) (Alignment Score)": null, "PartiPromts (ARNIQA)": 0.5982, "PartiPromts (ClipScore)": 27.5003, "PartiPromts (ClipIQA)": 0.8799, "PartiPromts (Sharpness - Laplacian Variance)": 5101.113, "HPS (v2.1)": 0.1973, "DrawBench (ClipScore)": 27.3007, "DrawBench (Image Reward)": 0.9612, "Median Inference Time": 3.862, "Price per Image": 0.025}
|
| 8 |
-
{"Platform": "Replicate", "Owner": "Black Forest Labs", "Device": "1xH100", "Model": "FLUX 1.1 Pro", "Optimization": "Undisclosed", "URL": "https://replicate.com/black-forest-labs/flux-1.1-pro", "PartiPromts (ARNIQA)": 0.5879, "PartiPromts (ClipScore)": 27.8015, "PartiPromts (ClipIQA)": 0.8273, "PartiPromts (Sharpness - Laplacian Variance)": 6773.8274, "HPS (v2.1)": 0.2048, "OneIG (Anime and Stylization) (Alignment Score)": 0.87, "OneIG (General Object) (Alignment Score)": 0.83, "OneIG (Portrait) (Alignment Score)": 0.78, "DrawBench (ClipScore)": 27.7958, "DrawBench (Image Reward)": 0.9258, "Long Text Bench (edit_distance)": 163.28, "Long Text Bench (text_word_accuracy)": null, "GenAI-Bench (VQA)": 0.7813, "Median Inference Time": 2.8571, "Price per Image": 0.04}
|
| 9 |
-
{"Platform": "Black Forest Labs", "Owner": "Black Forest Labs", "Device": "1xH100", "Model": "FLUX 2 pro", "Optimization": "Undisclosed", "URL": "https://bfl.ai/models", "PartiPromts (ARNIQA)": 0.6488, "PartiPromts (ClipScore)": 28.0808, "PartiPromts (ClipIQA)": 0.9172, "PartiPromts (Sharpness - Laplacian Variance)": 9335.3041, "GenAI-Bench (VQA)": 0.9633, "OneIG (Anime and Stylization) (Alignment Score)": null, "DrawBench (ClipScore)": 28.22, "DrawBench (Image Reward)": 1.0339, "Long Text Bench (edit_distance)": 159.2424, "Long Text Bench (text_word_accuracy)": null, "HPS (v2.1)": 0.1894, "Median Inference Time": 9.1787, "Price per Image": 0.1}
|
| 10 |
-
{"Platform": "fal.ai", "Owner": "fal.ai", "Device": "Undisclosed", "Model": "FLUX Krea", "Optimization": "Undisclosed", "URL": "https://fal.ai/models/fal-ai/flux/krea", "DrawBench (ClipScore)": 28.1482, "DrawBench (Image Reward)": 0.9853, "OneIG (Anime and Stylization) (Alignment Score)": null, "Long Text Bench (edit_distance)": 178.5125, "Long Text Bench (text_word_accuracy)": null, "GenAI-Bench (VQA)": 0.7949, "PartiPromts (ARNIQA)": 0.615, "PartiPromts (ClipScore)": 27.7512, "PartiPromts (ClipIQA)": 0.848, "PartiPromts (Sharpness - Laplacian Variance)": 4043.6115, "HPS (v2.1)": 0.1902, "Median Inference Time": 2.0193, "Price per Image": 0.025}
|
| 11 |
-
{"Platform": "Runware", "Owner": "Runware", "Device": "Undisclosed", "Model": "FLUX Dev", "Optimization": "Undisclosed", "URL": "https://runware.ai/models#", "PartiPromts (ARNIQA)": 0.638, "PartiPromts (ClipScore)": 27.3993, "PartiPromts (ClipIQA)": 0.8887, "PartiPromts (Sharpness - Laplacian Variance)": 6238.0671, "DrawBench (ClipScore)": 27.2074, "DrawBench (Image Reward)": 0.9923, "GenAI-Bench (VQA)": 0.7761, "Long Text Bench (edit_distance)": 178.3625, "Long Text Bench (text_word_accuracy)": null, "HPS (v2.1)": 0.194, "OneIG (Anime and Stylization) (Alignment Score)": null, "Median Inference Time": 4.2133, "Price per Image": 0.0038}
|
| 12 |
-
{"Platform": "Black Forest Labs", "Owner": "Black Forest Labs", "Device": "1xH100", "Model": "FLUX 2 beta", "Optimization": "Undisclosed", "URL": "https://bfl.ai/models", "DrawBench (ClipScore)": 28.2489, "DrawBench (Image Reward)": 1.0498, "PartiPromts (ARNIQA)": 0.5988, "PartiPromts (ClipScore)": 28.2471, "PartiPromts (ClipIQA)": 0.8992, "PartiPromts (Sharpness - Laplacian Variance)": 7910.49, "Long Text Bench (edit_distance)": 155.9429, "Long Text Bench (text_word_accuracy)": null, "OneIG (Anime and Stylization) (Alignment Score)": null, "HPS (v2.1)": 0.2121, "GenAI-Bench (VQA)": 0.9673, "Median Inference Time": 15.5713, "Price per Image": 0.025}
|
| 13 |
-
{"Platform": "Replicate", "Owner": "Black Forest Labs", "Device": "1xH100", "Model": "FLUX Schnell", "Optimization": "go_fast", "URL": "https://replicate.com/black-forest-labs/flux-schnell", "DrawBench (ClipScore)": 28.0363, "DrawBench (Image Reward)": 0.9009, "GenAI-Bench (VQA)": 0.7765, "OneIG (Anime and Stylization) (Alignment Score)": 0.88, "OneIG (Portrait) (Alignment Score)": 0.79, "OneIG (General Object) (Alignment Score)": 0.83, "PartiPromts (ARNIQA)": 0.5714, "PartiPromts (ClipScore)": 27.9771, "PartiPromts (ClipIQA)": 0.7947, "PartiPromts (Sharpness - Laplacian Variance)": 5685.5457, "HPS (v2.1)": 0.2086, "Long Text Bench (edit_distance)": 178.5125, "Long Text Bench (text_word_accuracy)": null, "Median Inference Time": 0.979, "Price per Image": 0.003}
|
| 14 |
-
{"Platform": "Replicate", "Owner": "Black Forest Labs", "Device": "1xH100", "Model": "FLUX Krea", "Optimization": "go_fast", "URL": "https://replicate.com/black-forest-labs/flux-krea-dev", "DrawBench (ClipScore)": 28.3217, "DrawBench (Image Reward)": 0.9533, "PartiPromts (ARNIQA)": 0.6336, "PartiPromts (ClipScore)": 27.88, "PartiPromts (ClipIQA)": 0.8167, "PartiPromts (Sharpness - Laplacian Variance)": 4121.4541, "HPS (v2.1)": 0.1967, "OneIG (Anime and Stylization) (Alignment Score)": 0.88, "OneIG (Portrait) (Alignment Score)": 0.81, "OneIG (General Object) (Alignment Score)": 0.84, "GenAI-Bench (VQA)": 0.8218, "Long Text Bench (edit_distance)": 178.5125, "Long Text Bench (text_word_accuracy)": null, "Median Inference Time": 1.8774, "Price per Image": 0.025}
|
| 15 |
-
{"Platform": "Prodia", "Owner": "Prodia", "Device": "Undisclosed", "Model": "FLUX Schnell", "Optimization": "fast", "URL": "https://app.prodia.com/models", "PartiPromts (ARNIQA)": 0.5726, "PartiPromts (ClipScore)": 27.388, "PartiPromts (ClipIQA)": 0.8827, "PartiPromts (Sharpness - Laplacian Variance)": 5464.0814, "OneIG (Anime and Stylization) (Alignment Score)": null, "Long Text Bench (edit_distance)": 178.5125, "Long Text Bench (text_word_accuracy)": null, "DrawBench (ClipScore)": 27.7805, "DrawBench (Image Reward)": 0.9845, "GenAI-Bench (VQA)": 0.7878, "HPS (v2.1)": 0.2128, "Median Inference Time": 0.7472, "Price per Image": 0.0015}
|
| 16 |
-
{"Platform": "Replicate", "Owner": "Black Forest Labs", "Device": "1xH100", "Model": "FLUX Dev", "Optimization": "go_fast", "URL": "https://replicate.com/black-forest-labs/flux-dev", "PartiPromts (ARNIQA)": 0.5764, "PartiPromts (ClipScore)": 27.8144, "PartiPromts (ClipIQA)": 0.7859, "PartiPromts (Sharpness - Laplacian Variance)": 5164.9466, "Long Text Bench (edit_distance)": 177.8931, "Long Text Bench (text_word_accuracy)": null, "DrawBench (ClipScore)": 27.7887, "DrawBench (Image Reward)": 1.0318, "OneIG (Anime and Stylization) (Alignment Score)": 0.85, "OneIG (Portrait) (Alignment Score)": 0.78, "OneIG (General Object) (Alignment Score)": 0.8, "HPS (v2.1)": 0.1982, "GenAI-Bench (VQA)": 0.7686, "Median Inference Time": 1.8018, "Price per Image": 0.025}
|
| 17 |
-
{"Platform": "Together AI", "Owner": "Black Forest Labs", "Device": "Undisclosed", "Model": "FLUX Schnell", "Optimization": "schnell", "URL": "https://www.together.ai/models/flux-1-schnell", "PartiPromts (ARNIQA)": 0.6082, "PartiPromts (ClipScore)": 27.6475, "PartiPromts (ClipIQA)": 0.898, "PartiPromts (Sharpness - Laplacian Variance)": 6446.8138, "DrawBench (ClipScore)": 27.6897, "DrawBench (Image Reward)": 0.9301, "GenAI-Bench (VQA)": 0.7706, "OneIG (Anime and Stylization) (Alignment Score)": null, "HPS (v2.1)": 0.2058, "Long Text Bench (edit_distance)": 178.5125, "Long Text Bench (text_word_accuracy)": null, "Median Inference Time": 1.3602, "Price per Image": 0.003}
|
| 18 |
-
{"Platform": "Runware", "Owner": "Black Forest Labs", "Device": "Undisclosed", "Model": "FLUX 1.1 Pro", "Optimization": "Undisclosed", "URL": "https://runware.ai/models#", "PartiPromts (ARNIQA)": 0.659, "PartiPromts (ClipScore)": 27.4929, "PartiPromts (ClipIQA)": 0.9074, "PartiPromts (Sharpness - Laplacian Variance)": 6874.5826, "Long Text Bench (edit_distance)": 167.1325, "Long Text Bench (text_word_accuracy)": null, "DrawBench (ClipScore)": 27.7755, "DrawBench (Image Reward)": 0.9277, "HPS (v2.1)": 0.1971, "OneIG (Anime and Stylization) (Alignment Score)": null, "GenAI-Bench (VQA)": 0.7723, "Median Inference Time": 4.2135, "Price per Image": 0.04}
|
| 19 |
-
{"Platform": "Runware", "Owner": "Runware", "Device": "Undisclosed", "Model": "FLUX Schnell", "Optimization": "Undisclosed", "URL": "https://runware.ai/models#", "HPS (v2.1)": 0.2026, "PartiPromts (ARNIQA)": 0.6344, "PartiPromts (ClipScore)": 27.7251, "PartiPromts (ClipIQA)": 0.8873, "PartiPromts (Sharpness - Laplacian Variance)": 7144.7132, "GenAI-Bench (VQA)": 0.769, "OneIG (Anime and Stylization) (Alignment Score)": null, "DrawBench (ClipScore)": 27.9708, "DrawBench (Image Reward)": 0.9571, "Long Text Bench (edit_distance)": 178.3812, "Long Text Bench (text_word_accuracy)": 0.0005, "Median Inference Time": 1.7584, "Price per Image": 0.0013}
|
| 20 |
-
{"Platform": "Together AI", "Owner": "Black Forest Labs", "Device": "Undisclosed", "Model": "FLUX Krea", "Optimization": "dev", "URL": "https://www.together.ai/models/flux-1-krea-dev", "Long Text Bench (edit_distance)": 178.5125, "Long Text Bench (text_word_accuracy)": null, "DrawBench (ClipScore)": 28.2254, "DrawBench (Image Reward)": 1.101, "GenAI-Bench (VQA)": 0.797, "PartiPromts (ARNIQA)": 0.618, "PartiPromts (ClipScore)": 27.6353, "PartiPromts (ClipIQA)": 0.8776, "PartiPromts (Sharpness - Laplacian Variance)": 5297.8361, "OneIG (Anime and Stylization) (Alignment Score)": null, "HPS (v2.1)": 0.1981, "Median Inference Time": 4.3831, "Price per Image": 0.025}
|
| 21 |
-
{"Platform": "Bria", "Owner": "Bria", "Device": "Undisclosed", "Model":"FIBO", Optimization: "Undisclosed", "URL": "https://bria.ai/models", "OneIG (Anime and Stylization) (Alignment Score)": 0.87, "Median Inference Time": 15.8328 , "Price per Image": 0.04}
|
| 22 |
-
{"Platform": "Replicate", "Owner": "Black Forest Labs", "Device": "Undisclosed", "Model": "FLUX 1.1 Pro Ultra", "Optimization": "Undisclosed", "URL": "", "OneIG (Anime and Stylization) (Alignment Score)": 0.89, "OneIG (Portrait) (Alignment Score)": 0.8, "OneIG (General Object) (Alignment Score)": 0.85}
|
| 23 |
-
{"Platform": "Replicate", "Owner": "Black Forest Labs", "Device": "Undisclosed", "Model": "FLUX 2 Pro", "Optimization": "", URL: "", "OneIG (Anime and Stylization) (Alignment Score)": 0.92, "OneIG (Portrait) (Alignment Score)": 0.84, "OneIG (General Object) (Alignment Score)": 0.88}
|
| 24 |
-
{"Platform": "Replicate", "Owner": "Black Forest Labs", "Device": "Undisclosed", "Model": "FLUX 2 Flex", "Optimization": "", URL: "", "OneIG (Anime and Stylization) (Alignment Score)": 0.92, "OneIG (Portrait) (Alignment Score)": 0.84, "OneIG (General Object) (Alignment Score)": 0.90}
|
| 25 |
-
{"Platform": "Replicate", "Owner": "Black Forest Labs", "Device": "Undisclosed", "Model": "Flux 2 Max", "Optimization": "", URL: "", "OneIG (Anime and Stylization) (Alignment Score)": 0.93, "OneIG (Portrait) (Alignment Score)": 0.85, "OneIG (General Object) (Alignment Score)": 0.89}
|
| 26 |
-
{"Platform": "Replicate", "Owner": "Black Forest Labs", "Device": "Undisclosed", "Model": "Flux 2 Pro", "Optimization": "", URL: "", "OneIG (Anime and Stylization) (Alignment Score)": null, "OneIG (Portrait) (Alignment Score)": 0.84, "OneIG (General Object) (Alignment Score)": 0.90}
|
| 27 |
-
{"Platform": "Replicate", "Owner": "Open AI", "Device": "Undisclosed", "Model": "GPT Image 1.5", "Optimization": "", URL: "", "OneIG (Anime and Stylization) (Alignment Score)": 0.92, "OneIG (Portrait) (Alignment Score)": 0.85, "OneIG (General Object) (Alignment Score)": null}
|
| 28 |
-
{"Platform": "Replicate", "Owner": "Pruna AI", "Device": "Undisclosed", "Model": "Hidream I1 Dev", "Optimzation": "Extra Juiced", "URL":"", "OneIG (Anime and Stylization) (Alignment Score)": 0.87, "OneIG (Portrait) (Alignment Score)": 0.79, "OneIG (General Object) (Alignment Score)": 0.82}
|
| 29 |
-
{"Platform": "Replicate", "Owner": "Pruna AI", "Device": "Undisclosed", "Model": "Hidream I1 Faast", "Optimzation": "Extra Juiced", "URL":"", "OneIG (Anime and Stylization) (Alignment Score)": 0.87, "OneIG (Portrait) (Alignment Score)": 0.79, "OneIG (General Object) (Alignment Score)": 0.82}
|
| 30 |
-
|
| 31 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@@ -0,0 +1,214 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""UI-only model display names.
|
| 2 |
+
|
| 3 |
+
Raw CSV / sample IDs stay unchanged; this maps aliases to a shared label
|
| 4 |
+
shown in leaderboards, Pareto plots, dropdowns, and sample headers.
|
| 5 |
+
"""
|
| 6 |
+
|
| 7 |
+
from __future__ import annotations
|
| 8 |
+
|
| 9 |
+
import re
|
| 10 |
+
|
| 11 |
+
# Explicit aliases (any leaderboard / sample id) -> shared display label.
|
| 12 |
+
MODEL_DISPLAY_NAMES = {
|
| 13 |
+
# FLUX
|
| 14 |
+
"flux_2_pro": "FLUX.2 [pro]",
|
| 15 |
+
"flux_2_max": "FLUX.2 [max]",
|
| 16 |
+
"flux_2_flex": "FLUX.2 [flex]",
|
| 17 |
+
"flux_2_dev": "FLUX.2 [dev]",
|
| 18 |
+
"flux_1_1_pro": "FLUX1.1 [pro]",
|
| 19 |
+
"flux_1_1_pro_ultra": "FLUX1.1 [pro] Ultra",
|
| 20 |
+
"flux_dev": "FLUX.1 [dev]",
|
| 21 |
+
"flux_schnell": "FLUX.1 [schnell]",
|
| 22 |
+
"flux_krea": "FLUX.1 Krea [dev]",
|
| 23 |
+
# GPT Image
|
| 24 |
+
"gpt_image_2": "GPT Image 2",
|
| 25 |
+
"GPT Image 2 (high)": "GPT Image 2",
|
| 26 |
+
"gpt_image_1_5": "GPT Image 1.5",
|
| 27 |
+
"GPT Image 1.5 (high)": "GPT Image 1.5",
|
| 28 |
+
"gpt_image_1": "GPT Image 1",
|
| 29 |
+
"GPT Image 1 (high)": "GPT Image 1",
|
| 30 |
+
"GPT Image 1 Mini (medium)": "GPT Image 1 Mini",
|
| 31 |
+
# Nano Banana / Gemini
|
| 32 |
+
"nano_banana_2_0": "Nano Banana 2",
|
| 33 |
+
"nano_banana_2": "Nano Banana 2",
|
| 34 |
+
"Nano Banana 2 (Gemini 3.1 Flash Image Preview)": "Nano Banana 2",
|
| 35 |
+
"Nano Banana 2 Lite (Gemini 3.1 Flash Lite Image)": "Nano Banana 2 Lite",
|
| 36 |
+
"nano_banana_pro": "Nano Banana Pro",
|
| 37 |
+
"Nano Banana Pro (Gemini 3 Pro Image)": "Nano Banana Pro",
|
| 38 |
+
"Nano Banana (Gemini 2.5 Flash Image)": "Nano Banana",
|
| 39 |
+
# Seedream
|
| 40 |
+
"seedream_5_0": "Seedream 5.0",
|
| 41 |
+
"seedream_4_5": "Seedream 4.5",
|
| 42 |
+
"seedream_4_0": "Seedream 4.0",
|
| 43 |
+
"seedream_3": "Seedream 3.0",
|
| 44 |
+
# Qwen
|
| 45 |
+
"qwen_image": "Qwen Image",
|
| 46 |
+
"qwen_image_2_0_pro": "Qwen Image 2.0 Pro",
|
| 47 |
+
"Qwen Image 2.0 Pro (2026-04-22)": "Qwen Image 2.0 Pro",
|
| 48 |
+
"Qwen Image 2.0 (2026-03-03)": "Qwen Image 2.0",
|
| 49 |
+
"qwen_image_2512": "Qwen Image 2512",
|
| 50 |
+
"Qwen Image Max 2512": "Qwen Image 2512",
|
| 51 |
+
"qwen_image_fast": "Qwen Image Fast",
|
| 52 |
+
# Ideogram
|
| 53 |
+
"ideogram_4_0_quality": "Ideogram 4.0 Quality",
|
| 54 |
+
"Ideogram 4.0 (Quality)": "Ideogram 4.0 Quality",
|
| 55 |
+
"Ideogram 4.0 Fast (Quality)": "Ideogram 4.0 Fast Quality",
|
| 56 |
+
# Imagen
|
| 57 |
+
"imagen_4_0": "Imagen 4",
|
| 58 |
+
"imagen_4": "Imagen 4",
|
| 59 |
+
"Imagen 4 Standard": "Imagen 4",
|
| 60 |
+
"imagen_4_0_ultra": "Imagen 4 Ultra",
|
| 61 |
+
"imagen_4_ultra": "Imagen 4 Ultra",
|
| 62 |
+
"imagen_4_fast": "Imagen 4 Fast",
|
| 63 |
+
"Imagen 3 (v002)": "Imagen 3",
|
| 64 |
+
# HiDream
|
| 65 |
+
"hidream_i1_dev": "HiDream-I1 Dev",
|
| 66 |
+
"HiDream-I1-Dev": "HiDream-I1 Dev",
|
| 67 |
+
"hidream_i1_fast": "HiDream-I1 Fast",
|
| 68 |
+
"HiDream-I1-Fast": "HiDream-I1 Fast",
|
| 69 |
+
"hidream_i1_full": "HiDream-I1 Full",
|
| 70 |
+
"HiDream-O1-Image": "HiDream-O1",
|
| 71 |
+
"HiDream-O1-Image-1.5": "HiDream-O1 1.5",
|
| 72 |
+
"HiDream-O1-Image-Dev": "HiDream-O1 Dev",
|
| 73 |
+
# Reve
|
| 74 |
+
"reve_2_1": "Reve 2.1",
|
| 75 |
+
"Reve Image (Halfmoon)": "Reve Image",
|
| 76 |
+
# P-Image
|
| 77 |
+
"p_image": "P-Image",
|
| 78 |
+
"p_image_2_ideogram_very_low_1k": "P-Image-Ideogram Very Low 1K",
|
| 79 |
+
"p_image_2_ideogram_very_low_2k": "P-Image-Ideogram Very Low 2K",
|
| 80 |
+
"P-Image-Ideogram (Very Low)": "P-Image-Ideogram Very Low",
|
| 81 |
+
"p_image_2_ideogram_low_1k": "P-Image-Ideogram Low 1K",
|
| 82 |
+
"p_image_2_ideogram_low_2k": "P-Image-Ideogram Low 2K",
|
| 83 |
+
"P-Image-Ideogram (Low)": "P-Image-Ideogram Low",
|
| 84 |
+
"p_image_2_ideogram_medium_1k": "P-Image-Ideogram Medium 1K",
|
| 85 |
+
"p_image_2_ideogram_medium_2k": "P-Image-Ideogram Medium 2K",
|
| 86 |
+
"P-Image-Ideogram (Medium)": "P-Image-Ideogram Medium",
|
| 87 |
+
"p_image_2_ideogram_high_1k": "P-Image-Ideogram High 1K",
|
| 88 |
+
"p_image_2_ideogram_high_2k": "P-Image-Ideogram High 2K",
|
| 89 |
+
"P-Image-Ideogram (High)": "P-Image-Ideogram High",
|
| 90 |
+
# Others overlapping P-Bench
|
| 91 |
+
"z_image": "Z-Image",
|
| 92 |
+
"glm_image": "GLM-Image",
|
| 93 |
+
"hunyuanimage_3_0": "HunyuanImage 3.0",
|
| 94 |
+
"hunyuan_image_3": "HunyuanImage 3.0",
|
| 95 |
+
"HunyuanImage 3.0 (Fal)": "HunyuanImage 3.0",
|
| 96 |
+
"HunyuanImage 3.0 Instruct (Fal)": "HunyuanImage 3.0 Instruct",
|
| 97 |
+
"wan_2_2_image": "Wan 2.2 Image",
|
| 98 |
+
"Wan2.6 Text to Image": "Wan 2.6 Text to Image",
|
| 99 |
+
"kling_v2_1": "Kling v2.1",
|
| 100 |
+
"juggernaut_base_flux": "Juggernaut Base FLUX",
|
| 101 |
+
"juggernaut_pro_flux": "Juggernaut Pro FLUX",
|
| 102 |
+
"juggernaut_lightning_flux": "Juggernaut Lightning FLUX",
|
| 103 |
+
"bria_4_fibo": "Bria FIBO",
|
| 104 |
+
"sdxl": "SDXL 1.0",
|
| 105 |
+
"Stable Diffusion XL 1.0": "SDXL 1.0",
|
| 106 |
+
"sdxl_lightning": "SDXL Lightning",
|
| 107 |
+
# Arena AI (kebab / arena.ai ids)
|
| 108 |
+
"gpt-image-2 (medium)": "GPT Image 2",
|
| 109 |
+
"gpt-image-1.5-high-fidelity": "GPT Image 1.5",
|
| 110 |
+
"gpt-image-1": "GPT Image 1",
|
| 111 |
+
"gpt-image-1-mini": "GPT Image 1 Mini",
|
| 112 |
+
"mai-image-2.6-preview": "MAI-Image-2.6 Preview",
|
| 113 |
+
"mai-image-2.5": "MAI-Image-2.5",
|
| 114 |
+
"mai-image-2": "MAI-Image-2",
|
| 115 |
+
"mai-image-1": "MAI Image 1",
|
| 116 |
+
"grok-imagine-image-2.0 (low)": "Grok Imagine Image 2.0",
|
| 117 |
+
"grok-imagine-image": "Grok Imagine Image",
|
| 118 |
+
"grok-imagine-image-pro": "Grok Imagine Image Pro",
|
| 119 |
+
"reve-2.1": "Reve 2.1",
|
| 120 |
+
"reve-2.0": "Reve 2.0",
|
| 121 |
+
"reve-v1.5": "Reve 1.5",
|
| 122 |
+
"muse-image": "Muse Image",
|
| 123 |
+
"gemini-3.1-flash-image (nano-banana-2) [web-search]": "Nano Banana 2",
|
| 124 |
+
"gemini-3.1-flash-lite-image (nano-banana-2-lite)": "Nano Banana 2 Lite",
|
| 125 |
+
"gemini-3-pro-image-2k (nano-banana-pro)": "Nano Banana Pro 2K",
|
| 126 |
+
"gemini-3-pro-image-preview (nano-banana-pro)": "Nano Banana Pro",
|
| 127 |
+
"gemini-2.5-flash-image-preview (nano-banana)": "Nano Banana",
|
| 128 |
+
"gemini-2.0-flash-preview-image-generation": "Gemini 2.0 Flash Image",
|
| 129 |
+
"seedream-5.0-pro": "Seedream 5.0 Pro",
|
| 130 |
+
"seedream-5.0-lite": "Seedream 5.0 Lite",
|
| 131 |
+
"seedream-4.5": "Seedream 4.5",
|
| 132 |
+
"seedream-4-2k": "Seedream 4.0 2K",
|
| 133 |
+
"seedream-4-fal": "Seedream 4.0",
|
| 134 |
+
"seedream-4-high-res-fal": "Seedream 4.0 High Res",
|
| 135 |
+
"seedream-3": "Seedream 3.0",
|
| 136 |
+
"qwen-image-3.0-pro": "Qwen Image 3.0 Pro",
|
| 137 |
+
"qwen-image-2.0-pro-2026-06-22": "Qwen Image 2.0 Pro",
|
| 138 |
+
"qwen-image-2512": "Qwen Image 2512",
|
| 139 |
+
"qwen-image-prompt-extend": "Qwen Image Prompt Extend",
|
| 140 |
+
"qwen-image": "Qwen Image",
|
| 141 |
+
"uni-1.1-max": "Luma UNI 1 Max",
|
| 142 |
+
"uni-1.1": "Luma UNI 1",
|
| 143 |
+
"Cosmos3-Super-Text2Image (Agentic)": "Cosmos3 Super Text2Image (Agentic)",
|
| 144 |
+
"Cosmos3-Super-Text2Image": "Cosmos3 Super Text2Image",
|
| 145 |
+
"recraft-v4.1-utility-pro": "Recraft V4.1 Utility Pro",
|
| 146 |
+
"recraft-v4.1-pro": "Recraft V4.1 Pro",
|
| 147 |
+
"recraft-v4": "Recraft V4",
|
| 148 |
+
"recraft-v3": "Recraft V3",
|
| 149 |
+
"flux-2-max": "FLUX.2 [max]",
|
| 150 |
+
"flux-2-flex": "FLUX.2 [flex]",
|
| 151 |
+
"flux-2-pro": "FLUX.2 [pro]",
|
| 152 |
+
"flux-2-dev": "FLUX.2 [dev]",
|
| 153 |
+
"flux-2-klein-9b": "FLUX.2 [klein] 9B",
|
| 154 |
+
"flux-2-klein-4b": "FLUX.2 [klein] 4B",
|
| 155 |
+
"flux-1-kontext-max": "FLUX.1 Kontext [max]",
|
| 156 |
+
"flux-1-kontext-pro": "FLUX.1 Kontext [pro]",
|
| 157 |
+
"flux-1-kontext-dev": "FLUX.1 Kontext [dev]",
|
| 158 |
+
"flux-1.1-pro": "FLUX1.1 [pro]",
|
| 159 |
+
"flux-1-dev-fp8": "FLUX.1 [dev]",
|
| 160 |
+
"hunyuan-image-3.0": "HunyuanImage 3.0",
|
| 161 |
+
"imagen-ultra-4.0-generate-001": "Imagen 4 Ultra",
|
| 162 |
+
"imagen-4.0-generate-001": "Imagen 4",
|
| 163 |
+
"imagen-3.0-generate-002": "Imagen 3",
|
| 164 |
+
"wan2.6-t2i": "Wan 2.6 Text to Image",
|
| 165 |
+
"wan2.5-t2i-preview": "Wan 2.5 Preview",
|
| 166 |
+
"wan2.7-image-pro": "Wan 2.7 Pro",
|
| 167 |
+
"wan2.7-image": "Wan 2.7",
|
| 168 |
+
"ideogram-4.0-quality": "Ideogram 4.0 Quality",
|
| 169 |
+
"ideogram-v3-quality": "Ideogram 3.0",
|
| 170 |
+
"ideogram-v2": "Ideogram v2",
|
| 171 |
+
"krea-2-medium": "Krea 2 Medium",
|
| 172 |
+
"krea-2-turbo": "Krea 2 Medium Turbo",
|
| 173 |
+
"krea-2-large": "Krea 2 Large",
|
| 174 |
+
"hidream-o1-image": "HiDream-O1",
|
| 175 |
+
"z-image-turbo": "Z-Image Turbo",
|
| 176 |
+
"photon": "Luma Photon",
|
| 177 |
+
"p-image": "P-Image",
|
| 178 |
+
"runway-gen4": "Runway Gen-4 Image",
|
| 179 |
+
"lucid-origin": "Lucid Origin",
|
| 180 |
+
"glm-image": "GLM-Image",
|
| 181 |
+
"dall-e-3": "DALLE 3",
|
| 182 |
+
"stable-diffusion-v35-large": "Stable Diffusion 3.5 Large",
|
| 183 |
+
"bagel": "Bagel",
|
| 184 |
+
}
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
def _prettify_snake_case(model_id: str) -> str:
|
| 188 |
+
parts = [part for part in str(model_id).split("_") if part]
|
| 189 |
+
pretty = []
|
| 190 |
+
for part in parts:
|
| 191 |
+
if part.isdigit():
|
| 192 |
+
pretty.append(part)
|
| 193 |
+
elif re.fullmatch(r"\d+k", part, flags=re.IGNORECASE):
|
| 194 |
+
pretty.append(part.upper())
|
| 195 |
+
elif part.lower() in {"flux", "gpt", "sdxl", "glm"}:
|
| 196 |
+
pretty.append(part.upper())
|
| 197 |
+
else:
|
| 198 |
+
pretty.append(part.capitalize())
|
| 199 |
+
return " ".join(pretty)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def display_model_name(model_id) -> str:
|
| 203 |
+
"""Return the shared UI label for a raw model id / source name."""
|
| 204 |
+
if model_id is None:
|
| 205 |
+
return ""
|
| 206 |
+
raw = str(model_id).strip()
|
| 207 |
+
if not raw:
|
| 208 |
+
return ""
|
| 209 |
+
if raw in MODEL_DISPLAY_NAMES:
|
| 210 |
+
return MODEL_DISPLAY_NAMES[raw]
|
| 211 |
+
# Already a human label (spaces / punctuation) — keep as-is.
|
| 212 |
+
if re.search(r"[\s.\[\]()]", raw):
|
| 213 |
+
return raw
|
| 214 |
+
return _prettify_snake_case(raw)
|
|
|
@@ -1,2 +1 @@
|
|
| 1 |
-
|
| 2 |
-
plotly
|
|
|
|
| 1 |
+
plotly
|
|
|
|
@@ -1,317 +1,416 @@
|
|
| 1 |
from html import escape
|
|
|
|
|
|
|
| 2 |
import random
|
| 3 |
|
| 4 |
import gradio as gr
|
| 5 |
import pandas as pd
|
| 6 |
import plotly.graph_objects as go
|
| 7 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
-
FILTER_COLUMNS = ["Platform", "Endpoint Owner", "Model", "Optimized"]
|
| 10 |
MAX_COMPARE_MODELS = 4
|
| 11 |
DEFAULT_COMPARE_PROMPTS = 3
|
| 12 |
MAX_COMPARE_PROMPTS = 8
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
"""
|
| 48 |
|
| 49 |
-
|
| 50 |
-
#
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
"""
|
| 55 |
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
"""
|
| 64 |
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
"""
|
| 71 |
|
| 72 |
-
RAPIDATA_ELO_SCORES_CONTENT = """
|
| 73 |
-
### Reported Rapidata Elo scores
|
| 74 |
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 79 |
|
| 80 |
-
RAPIDATA_ELO_RUN_DETAILS_CONTENT = """
|
| 81 |
-
### Reading the results
|
| 82 |
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
""
|
|
|
|
|
|
|
| 86 |
|
| 87 |
-
BENCHMARK_AI_ELO_SCORES_CONTENT = """
|
| 88 |
-
### Reported Benchmark.ai Elo scores
|
| 89 |
|
| 90 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 91 |
|
| 92 |
-
This score is not from the Qwen Image Bench prompt suite; it is shown alongside for
|
| 93 |
-
cross-benchmark comparison.
|
| 94 |
-
"""
|
| 95 |
|
| 96 |
-
|
| 97 |
-
|
|
|
|
| 98 |
|
| 99 |
-
Each row records generation time and price per image when available. Rankings for
|
| 100 |
-
this metric alone are ordered by **Benchmark.ai Elo**.
|
| 101 |
-
"""
|
| 102 |
|
| 103 |
-
|
| 104 |
-
|
|
|
|
|
|
|
| 105 |
|
| 106 |
-
InferBench compares **text-to-image models** on quality, preference, latency, and
|
| 107 |
-
price. Results are organized by prompt suite (benchmark), not by a single opaque
|
| 108 |
-
score.
|
| 109 |
|
| 110 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
|
| 112 |
-
- **Home** — snapshot of each prompt suite and unique model count.
|
| 113 |
-
- **Benchmarks** — open a prompt suite to see its metric columns, graphs (including a
|
| 114 |
-
quality-vs-price Pareto frontier), and side-by-side sample generations.
|
| 115 |
-
- **About** — this page.
|
| 116 |
|
| 117 |
-
|
| 118 |
|
| 119 |
-
### OneIG Alignment
|
| 120 |
-
The **alignment** slice of OneIG (not the full OneIG suite), across three categories:
|
| 121 |
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
|
| 126 |
-
The table reports category **alignment scores**, **Datapoint Elo** columns (Anime /
|
| 127 |
-
Human / Object), median / min generation time, and price per image. Rankings use the
|
| 128 |
-
mean of each model's available category alignment scores (missing categories are
|
| 129 |
-
skipped for that model).
|
| 130 |
|
| 131 |
-
|
| 132 |
-
|
|
|
|
|
|
|
|
|
|
| 133 |
|
| 134 |
-
- **P-Judge Overall** — automatic preference / quality score
|
| 135 |
-
- **Datapoint Elo** — human-preference Elo (default sort key)
|
| 136 |
-
- **Rapidata Elo** — Elo from the Rapidata evaluation on this suite
|
| 137 |
|
| 138 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
|
| 140 |
-
## How to read the numbers
|
| 141 |
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 146 |
|
| 147 |
-
The Pareto plot highlights models that are not dominated on both **higher score** and
|
| 148 |
-
**lower price**.
|
| 149 |
-
"""
|
| 150 |
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
not run (or not reported) for that model.
|
| 158 |
-
- Elo ratings can shift when the comparison pool changes — treat them as relative
|
| 159 |
-
rankings for the snapshot, not absolute constants.
|
| 160 |
-
- Optimized / accelerated endpoints (when labeled) may differ from the base model
|
| 161 |
-
publisher's default serving stack.
|
| 162 |
-
|
| 163 |
-
Built by [Pruna AI](https://www.pruna.ai/). Contributions and new evaluation runs welcome.
|
| 164 |
-
"""
|
| 165 |
|
| 166 |
-
COMMUNITY_CONTENT = """
|
| 167 |
-
<div class="community-footer">
|
| 168 |
-
<h3>Join the Pruna AI community</h3>
|
| 169 |
-
<div class="community-footer-links">
|
| 170 |
-
<a rel="nofollow" href="https://x.com/PrunaAI" target="_blank">X / Twitter</a>
|
| 171 |
-
<a rel="nofollow" href="https://github.com/PrunaAI/pruna" target="_blank">GitHub</a>
|
| 172 |
-
<a rel="nofollow" href="https://www.linkedin.com/company/pruna-ai" target="_blank">LinkedIn</a>
|
| 173 |
-
<a rel="nofollow" href="https://discord.com/invite/rskEr4BZJx" target="_blank">Discord</a>
|
| 174 |
-
<a rel="nofollow" href="https://www.reddit.com/r/PrunaAI/" target="_blank">Reddit</a>
|
| 175 |
-
<a rel="nofollow" href="https://www.pruna.ai/" target="_blank">pruna.ai</a>
|
| 176 |
-
</div>
|
| 177 |
-
</div>
|
| 178 |
-
"""
|
| 179 |
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 190 |
|
| 191 |
|
| 192 |
-
def
|
| 193 |
-
|
| 194 |
-
"""
|
| 195 |
-
<div style="text-align: center;">
|
| 196 |
-
<h1>InferBench</h1>
|
| 197 |
-
<h2>Compare text-to-image models on quality, speed, and price</h2>
|
| 198 |
-
</div>
|
| 199 |
-
"""
|
| 200 |
-
)
|
| 201 |
|
| 202 |
|
| 203 |
-
def
|
| 204 |
-
|
|
|
|
| 205 |
return []
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
.loc[lambda df: ~df["Model"].astype(str).str.startswith("#")]
|
| 210 |
-
.sort_values(score_column, ascending=False)
|
| 211 |
-
.head(n)
|
| 212 |
-
)
|
| 213 |
return [
|
| 214 |
-
|
| 215 |
-
for
|
|
|
|
| 216 |
]
|
| 217 |
|
| 218 |
|
| 219 |
-
|
| 220 |
-
""
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
highlights.append(
|
| 239 |
-
{
|
| 240 |
-
"label": f"BEST {benchmark['title'].upper()}",
|
| 241 |
-
"model": model,
|
| 242 |
-
"detail": f"{_display_label(score_column)} · {_format_score(score)}",
|
| 243 |
-
}
|
| 244 |
-
)
|
| 245 |
-
|
| 246 |
-
if unique_models:
|
| 247 |
-
highlights.append(
|
| 248 |
-
{
|
| 249 |
-
"label": "MODELS SCORED",
|
| 250 |
-
"model": str(len(unique_models)),
|
| 251 |
-
"detail": "unique across prompt suites",
|
| 252 |
-
}
|
| 253 |
-
)
|
| 254 |
-
return highlights
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
def render_home(benchmarks):
|
| 258 |
-
highlights = _home_highlights(benchmarks)
|
| 259 |
-
|
| 260 |
-
gr.Markdown(
|
| 261 |
-
"""
|
| 262 |
-
InferBench is organized by **prompt suite**. There is no single global
|
| 263 |
-
“best model” score — open a benchmark for full tables, graphs, and sample
|
| 264 |
-
comparisons.
|
| 265 |
-
"""
|
| 266 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 267 |
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 274 |
]
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
score_column = score_columns[0] if score_columns else None
|
| 285 |
-
top = _top_models(data, score_column, n=3) if score_column else []
|
| 286 |
-
score_label = _display_label(score_column) if score_column else "Score"
|
| 287 |
-
rows_html = "".join(
|
| 288 |
-
f"<li><span class='home-rank'>{idx}</span>"
|
| 289 |
-
f"<span class='home-model'>{escape(model)}</span>"
|
| 290 |
-
f"<span class='home-score'>{_format_score(score)}</span></li>"
|
| 291 |
-
for idx, (model, score) in enumerate(top, start=1)
|
| 292 |
-
) or "<li class='home-empty'>No scores yet.</li>"
|
| 293 |
-
|
| 294 |
-
with gr.Column(scale=1, min_width=280):
|
| 295 |
-
gr.HTML(
|
| 296 |
-
f"""
|
| 297 |
-
<div class="home-benchmark-card">
|
| 298 |
-
<div class="home-benchmark-title">
|
| 299 |
-
{escape(benchmark.get("emoji", "📊"))}
|
| 300 |
-
{escape(benchmark["title"])}
|
| 301 |
-
</div>
|
| 302 |
-
<p class="home-benchmark-blurb">
|
| 303 |
-
{escape(benchmark.get("card_description", ""))}
|
| 304 |
-
</p>
|
| 305 |
-
<div class="home-top-label">Top 3 by {escape(score_label)}</div>
|
| 306 |
-
<ol class="home-top-list">{rows_html}</ol>
|
| 307 |
-
</div>
|
| 308 |
-
"""
|
| 309 |
-
)
|
| 310 |
|
| 311 |
|
| 312 |
def _format_leaderboard_cell(column, value):
|
| 313 |
if pd.isna(value) or value is None or value == "":
|
| 314 |
-
return "
|
| 315 |
label = str(column).lower()
|
| 316 |
if label == "rank":
|
| 317 |
return str(int(value))
|
|
@@ -323,6 +422,8 @@ def _format_leaderboard_cell(column, value):
|
|
| 323 |
except (TypeError, ValueError):
|
| 324 |
return escape(str(value))
|
| 325 |
if label in {"model", "platform", "endpoint owner", "optimized"}:
|
|
|
|
|
|
|
| 326 |
return escape(str(value))
|
| 327 |
try:
|
| 328 |
number = float(value)
|
|
@@ -353,10 +454,34 @@ def _leaderboard_sort_type(column):
|
|
| 353 |
return "number"
|
| 354 |
|
| 355 |
|
| 356 |
-
def
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
if leaderboard.empty:
|
| 361 |
return (
|
| 362 |
'<div class="ranking-table-scroll">'
|
|
@@ -367,18 +492,19 @@ def _leaderboard_html(data, columns, score_columns, overall_column):
|
|
| 367 |
header_cells = []
|
| 368 |
for index, column in enumerate(leaderboard.columns):
|
| 369 |
sort_type = _leaderboard_sort_type(column)
|
|
|
|
| 370 |
header_cells.append(
|
| 371 |
-
f'<th class="sortable-col
|
| 372 |
-
f'title="Sort by {escape(str(column))}">
|
|
|
|
| 373 |
)
|
| 374 |
body_rows = []
|
| 375 |
for _, row in leaderboard.iterrows():
|
| 376 |
cells = []
|
| 377 |
for column in leaderboard.columns:
|
| 378 |
-
css =
|
| 379 |
sort_value = escape(_leaderboard_sort_value(column, row[column]), quote=True)
|
| 380 |
if column == "Model":
|
| 381 |
-
css = "model-cell"
|
| 382 |
cells.append(
|
| 383 |
f'<td class="{css}" data-sort-value="{sort_value}">'
|
| 384 |
f"<strong>{_format_leaderboard_cell(column, row[column])}</strong></td>"
|
|
@@ -402,130 +528,17 @@ def _leaderboard_html(data, columns, score_columns, overall_column):
|
|
| 402 |
"""
|
| 403 |
|
| 404 |
|
| 405 |
-
def render_leaderboard(
|
| 406 |
-
data,
|
| 407 |
-
columns,
|
| 408 |
-
note=None,
|
| 409 |
-
score_columns=None,
|
| 410 |
-
overall_column=None,
|
| 411 |
-
):
|
| 412 |
-
score_columns = list(score_columns or _infer_score_columns(columns))
|
| 413 |
-
overall_column = overall_column or _default_overall_column(score_columns)
|
| 414 |
-
platform_choices = _filter_choices(data, "Platform")
|
| 415 |
-
owner_choices = _filter_choices(data, "Endpoint Owner")
|
| 416 |
-
optimized_choices = _filter_choices(data, "Optimized")
|
| 417 |
-
|
| 418 |
-
if note:
|
| 419 |
-
gr.Markdown(note)
|
| 420 |
-
|
| 421 |
-
filter_inputs = []
|
| 422 |
-
with gr.Row(elem_classes="leaderboard-controls"):
|
| 423 |
-
search = gr.Textbox(
|
| 424 |
-
label="Search models",
|
| 425 |
-
placeholder="Search by model or provider",
|
| 426 |
-
scale=3,
|
| 427 |
-
)
|
| 428 |
-
filter_inputs.append(search)
|
| 429 |
-
platform = None
|
| 430 |
-
owner = None
|
| 431 |
-
optimized = None
|
| 432 |
-
if platform_choices:
|
| 433 |
-
platform = gr.Dropdown(
|
| 434 |
-
choices=platform_choices,
|
| 435 |
-
value=[],
|
| 436 |
-
label="Providers",
|
| 437 |
-
multiselect=True,
|
| 438 |
-
scale=1,
|
| 439 |
-
)
|
| 440 |
-
filter_inputs.append(platform)
|
| 441 |
-
if owner_choices:
|
| 442 |
-
owner = gr.Dropdown(
|
| 443 |
-
choices=owner_choices,
|
| 444 |
-
value=[],
|
| 445 |
-
label="Endpoint owners",
|
| 446 |
-
multiselect=True,
|
| 447 |
-
scale=1,
|
| 448 |
-
)
|
| 449 |
-
filter_inputs.append(owner)
|
| 450 |
-
if optimized_choices:
|
| 451 |
-
optimized = gr.Dropdown(
|
| 452 |
-
choices=optimized_choices,
|
| 453 |
-
value=[],
|
| 454 |
-
label="Optimized",
|
| 455 |
-
multiselect=True,
|
| 456 |
-
scale=1,
|
| 457 |
-
)
|
| 458 |
-
filter_inputs.append(optimized)
|
| 459 |
-
|
| 460 |
-
ranking = gr.HTML(
|
| 461 |
-
_leaderboard_html(data, columns, score_columns, overall_column),
|
| 462 |
-
elem_classes="ranking-table-host",
|
| 463 |
-
)
|
| 464 |
-
|
| 465 |
-
def update_ranking(
|
| 466 |
-
search_term,
|
| 467 |
-
platform_value=None,
|
| 468 |
-
owner_value=None,
|
| 469 |
-
optimized_value=None,
|
| 470 |
-
):
|
| 471 |
-
filtered_data = _filter_leaderboard(
|
| 472 |
-
data,
|
| 473 |
-
search_term,
|
| 474 |
-
platform_value or [],
|
| 475 |
-
owner_value or [],
|
| 476 |
-
optimized_value or [],
|
| 477 |
-
)
|
| 478 |
-
return _leaderboard_html(
|
| 479 |
-
filtered_data, columns, score_columns, overall_column
|
| 480 |
-
)
|
| 481 |
-
|
| 482 |
-
# Wire only the filters that actually exist for this table.
|
| 483 |
-
change_inputs = [search]
|
| 484 |
-
if platform is not None:
|
| 485 |
-
change_inputs.append(platform)
|
| 486 |
-
if owner is not None:
|
| 487 |
-
change_inputs.append(owner)
|
| 488 |
-
if optimized is not None:
|
| 489 |
-
change_inputs.append(optimized)
|
| 490 |
-
|
| 491 |
-
for component in filter_inputs:
|
| 492 |
-
component.change(
|
| 493 |
-
update_ranking,
|
| 494 |
-
inputs=change_inputs,
|
| 495 |
-
outputs=ranking,
|
| 496 |
-
)
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
def _infer_score_columns(columns):
|
| 500 |
-
return [column for column in columns if column.startswith("OneIG (")]
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
def _default_overall_column(score_columns):
|
| 504 |
-
if len(score_columns) == 1:
|
| 505 |
-
return score_columns[0]
|
| 506 |
-
return "OneIG Overall Score"
|
| 507 |
-
|
| 508 |
-
|
| 509 |
def _filter_choices(data, column):
|
| 510 |
-
if column not in data.columns:
|
| 511 |
return []
|
| 512 |
return sorted(data[column].dropna().astype(str).unique().tolist())
|
| 513 |
|
| 514 |
|
| 515 |
-
def _filter_leaderboard(data,
|
| 516 |
filtered = data.copy()
|
| 517 |
-
if
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
for column in ["Model", "Platform", "Endpoint Owner"]
|
| 521 |
-
if column in filtered.columns
|
| 522 |
-
]
|
| 523 |
-
matches = pd.Series(False, index=filtered.index)
|
| 524 |
-
for column in search_columns:
|
| 525 |
-
matches |= filtered[column].astype(str).str.contains(
|
| 526 |
-
search_term, case=False, na=False
|
| 527 |
-
)
|
| 528 |
-
filtered = filtered[matches]
|
| 529 |
|
| 530 |
for column, values in [
|
| 531 |
("Platform", platform),
|
|
@@ -537,64 +550,22 @@ def _filter_leaderboard(data, search_term, platform, owner, optimized):
|
|
| 537 |
return filtered
|
| 538 |
|
| 539 |
|
| 540 |
-
def _leaderboard_dataframe(data, columns
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
"Median Generation Time (s)",
|
| 553 |
-
"Min Generation Time (s)",
|
| 554 |
-
"Price / Image (USD)",
|
| 555 |
-
"Evaluation Date (UTC)",
|
| 556 |
-
"Date",
|
| 557 |
-
]
|
| 558 |
-
if column in data.columns
|
| 559 |
-
]
|
| 560 |
-
# Keep overall_column visible when the caller includes it (e.g. Datapoint Elo).
|
| 561 |
-
# Synthetic aggregates like OneIG Overall Score are simply omitted from `columns`.
|
| 562 |
-
middle = [
|
| 563 |
-
column
|
| 564 |
-
for column in columns
|
| 565 |
-
if column in data.columns
|
| 566 |
-
and column not in skip_columns
|
| 567 |
-
and column not in preferred_prefix
|
| 568 |
-
and column not in preferred_suffix
|
| 569 |
-
]
|
| 570 |
-
|
| 571 |
-
ordered_columns = []
|
| 572 |
-
seen = set()
|
| 573 |
-
for column in [*preferred_prefix, *middle, *preferred_suffix]:
|
| 574 |
-
if column not in seen:
|
| 575 |
-
seen.add(column)
|
| 576 |
-
ordered_columns.append(column)
|
| 577 |
-
|
| 578 |
-
leaderboard = data[ordered_columns].copy()
|
| 579 |
-
|
| 580 |
-
# Rank by overall when available, even if that column is not displayed.
|
| 581 |
-
if overall_column and overall_column in data.columns:
|
| 582 |
-
leaderboard = (
|
| 583 |
-
leaderboard.assign(_sort_key=data[overall_column])
|
| 584 |
-
.sort_values("_sort_key", ascending=False, na_position="last")
|
| 585 |
-
.drop(columns=["_sort_key"])
|
| 586 |
-
.reset_index(drop=True)
|
| 587 |
-
)
|
| 588 |
-
else:
|
| 589 |
-
leaderboard = leaderboard.reset_index(drop=True)
|
| 590 |
-
|
| 591 |
-
leaderboard.insert(0, "Rank", leaderboard.index + 1)
|
| 592 |
-
return leaderboard.rename(columns=_display_label)
|
| 593 |
|
| 594 |
|
| 595 |
def _display_label(column):
|
| 596 |
labels = {
|
| 597 |
-
"_overall_score": "Overall score",
|
| 598 |
"OneIG Overall Score": "Overall",
|
| 599 |
"OneIG (Anime Alignment)": "Anime",
|
| 600 |
"OneIG (Human Alignment)": "Human",
|
|
@@ -602,10 +573,18 @@ def _display_label(column):
|
|
| 602 |
"OneIG Anime Elo": "Anime Elo (Datapoint)",
|
| 603 |
"OneIG Human Elo": "Human Elo (Datapoint)",
|
| 604 |
"OneIG Object Elo": "Object Elo (Datapoint)",
|
| 605 |
-
"P-Judge Overall": "P-
|
| 606 |
"Datapoint Elo": "Datapoint Elo",
|
| 607 |
"Rapidata Elo": "Rapidata Elo",
|
| 608 |
-
"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 609 |
"Raw Win Rate": "Raw win rate",
|
| 610 |
"Median Generation Time (s)": "Median generation time",
|
| 611 |
"Min Generation Time (s)": "Min generation time",
|
|
@@ -616,137 +595,325 @@ def _display_label(column):
|
|
| 616 |
return labels.get(column, column)
|
| 617 |
|
| 618 |
|
| 619 |
-
def
|
| 620 |
-
return "
|
| 621 |
|
| 622 |
|
| 623 |
-
def
|
| 624 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 625 |
|
| 626 |
|
| 627 |
-
def
|
| 628 |
-
return
|
|
|
|
|
|
|
|
|
|
|
|
|
| 629 |
|
| 630 |
|
| 631 |
-
def
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 635 |
|
| 636 |
-
|
| 637 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 638 |
)
|
| 639 |
-
with gr.Tabs(elem_classes="subtabs", selected=0) as detail_tabs:
|
| 640 |
-
with gr.TabItem("Leaderboard"):
|
| 641 |
-
render_leaderboard(
|
| 642 |
-
benchmark["data"],
|
| 643 |
-
benchmark["columns"],
|
| 644 |
-
note=benchmark.get("note"),
|
| 645 |
-
score_columns=benchmark.get("score_columns"),
|
| 646 |
-
overall_column=benchmark.get("overall_column"),
|
| 647 |
-
)
|
| 648 |
-
with gr.TabItem("Graphs"):
|
| 649 |
-
render_benchmark_graphs(benchmark)
|
| 650 |
-
with gr.TabItem("Compare samples"):
|
| 651 |
-
render_compare_samples(benchmark)
|
| 652 |
-
return detail_tabs
|
| 653 |
|
| 654 |
|
| 655 |
-
def
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
|
|
|
| 661 |
|
| 662 |
-
When generations are linked, you will be able to pick models and browse
|
| 663 |
-
side-by-side outputs for the same prompts.
|
| 664 |
-
"""
|
| 665 |
-
)
|
| 666 |
-
return
|
| 667 |
|
| 668 |
-
|
| 669 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 670 |
|
| 671 |
-
|
| 672 |
-
|
| 673 |
-
|
| 674 |
-
|
| 675 |
-
|
| 676 |
-
|
| 677 |
-
|
| 678 |
-
""
|
|
|
|
|
|
|
| 679 |
)
|
| 680 |
-
|
| 681 |
-
|
| 682 |
-
|
| 683 |
-
|
| 684 |
-
|
| 685 |
-
|
| 686 |
-
|
| 687 |
-
|
| 688 |
-
|
| 689 |
-
|
| 690 |
-
|
| 691 |
-
|
| 692 |
-
|
| 693 |
-
|
| 694 |
-
|
| 695 |
-
|
| 696 |
-
|
|
|
|
| 697 |
)
|
| 698 |
-
|
| 699 |
-
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
| 703 |
-
|
| 704 |
-
|
| 705 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 706 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 707 |
)
|
| 708 |
-
|
| 709 |
-
|
| 710 |
-
|
| 711 |
-
|
| 712 |
-
|
| 713 |
-
|
| 714 |
-
|
| 715 |
-
|
| 716 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 717 |
|
| 718 |
-
def shuffle_gallery(selected_models, num_prompts, seed):
|
| 719 |
-
next_seed = int(seed or 0) + 1
|
| 720 |
-
return next_seed, _build_compare_samples_html(
|
| 721 |
-
samples,
|
| 722 |
-
selected_models,
|
| 723 |
-
int(num_prompts),
|
| 724 |
-
seed=next_seed,
|
| 725 |
-
)
|
| 726 |
|
| 727 |
-
|
| 728 |
-
|
| 729 |
-
|
| 730 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 731 |
)
|
| 732 |
-
|
| 733 |
-
|
| 734 |
-
|
| 735 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 736 |
)
|
| 737 |
-
|
| 738 |
-
|
| 739 |
-
|
| 740 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 741 |
)
|
| 742 |
|
| 743 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 744 |
def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
|
| 745 |
-
selected_models = [
|
| 746 |
-
model
|
| 747 |
-
for model in (selected_models or [])
|
| 748 |
-
if model in samples["images"]
|
| 749 |
-
][:MAX_COMPARE_MODELS]
|
| 750 |
|
| 751 |
if not selected_models:
|
| 752 |
return (
|
|
@@ -787,9 +954,9 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
|
|
| 787 |
cells.append(
|
| 788 |
f"""
|
| 789 |
<div class="compare-cell">
|
| 790 |
-
<div class="compare-model-label">{escape(model)}</div>
|
| 791 |
<a href="{image_url}" target="_blank" rel="noopener noreferrer">
|
| 792 |
-
<img src="{image_url}" alt="{escape(model)} sample" loading="lazy" />
|
| 793 |
</a>
|
| 794 |
</div>
|
| 795 |
"""
|
|
@@ -812,312 +979,1013 @@ def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
|
|
| 812 |
return "\n".join(blocks)
|
| 813 |
|
| 814 |
|
| 815 |
-
def
|
| 816 |
-
|
| 817 |
-
|
| 818 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 819 |
|
| 820 |
-
with gr.Column(visible=True, elem_classes="benchmark-catalogue") as catalogue:
|
| 821 |
-
gr.Markdown(
|
| 822 |
-
"""
|
| 823 |
-
# Benchmarks
|
| 824 |
|
| 825 |
-
|
| 826 |
-
|
| 827 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 828 |
)
|
| 829 |
-
|
| 830 |
-
|
| 831 |
-
|
| 832 |
-
|
| 833 |
-
|
| 834 |
-
|
| 835 |
-
|
| 836 |
-
|
| 837 |
-
|
| 838 |
-
|
| 839 |
-
|
| 840 |
-
|
| 841 |
-
|
| 842 |
-
|
| 843 |
-
|
| 844 |
-
|
| 845 |
-
|
| 846 |
-
|
| 847 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 848 |
|
| 849 |
-
for benchmark in benchmarks:
|
| 850 |
-
with gr.Column(visible=False) as detail:
|
| 851 |
-
back_button = gr.Button("← All benchmarks", size="sm")
|
| 852 |
-
render_benchmark_detail(benchmark)
|
| 853 |
-
detail_entries.append((benchmark["id"], detail, back_button))
|
| 854 |
|
| 855 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 856 |
|
| 857 |
-
|
| 858 |
-
|
| 859 |
-
|
| 860 |
-
|
| 861 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 862 |
|
| 863 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 864 |
return (
|
| 865 |
-
|
| 866 |
-
|
| 867 |
-
|
| 868 |
-
|
| 869 |
-
|
|
|
|
|
|
|
|
|
|
| 870 |
)
|
| 871 |
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
| 875 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 876 |
)
|
| 877 |
|
| 878 |
-
|
| 879 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 880 |
|
| 881 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 882 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 883 |
|
| 884 |
-
def
|
| 885 |
-
|
| 886 |
-
|
| 887 |
-
|
| 888 |
-
|
| 889 |
-
|
| 890 |
-
|
| 891 |
-
|
| 892 |
-
|
| 893 |
-
|
| 894 |
-
|
| 895 |
-
|
| 896 |
-
|
| 897 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 898 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 899 |
|
| 900 |
-
|
| 901 |
-
|
| 902 |
-
|
| 903 |
-
|
| 904 |
-
|
| 905 |
-
|
| 906 |
-
|
| 907 |
-
):
|
| 908 |
-
|
| 909 |
-
data[["Model", score_column, x_column]]
|
| 910 |
-
.dropna()
|
| 911 |
-
.copy()
|
| 912 |
-
.reset_index(drop=True)
|
| 913 |
-
)
|
| 914 |
-
if scatter.empty:
|
| 915 |
-
return None
|
| 916 |
|
| 917 |
-
|
| 918 |
-
|
| 919 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 920 |
|
| 921 |
-
|
| 922 |
-
|
| 923 |
-
|
| 924 |
-
|
| 925 |
-
|
| 926 |
-
|
| 927 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 928 |
|
| 929 |
-
|
| 930 |
-
|
| 931 |
-
|
| 932 |
-
|
| 933 |
-
|
| 934 |
-
|
| 935 |
-
|
| 936 |
-
|
| 937 |
-
|
| 938 |
-
|
| 939 |
-
|
| 940 |
-
|
| 941 |
-
|
| 942 |
-
|
| 943 |
-
|
| 944 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 945 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 946 |
)
|
| 947 |
-
|
| 948 |
-
|
| 949 |
-
|
| 950 |
-
|
| 951 |
-
|
| 952 |
-
|
| 953 |
-
|
| 954 |
-
|
| 955 |
-
|
| 956 |
-
|
| 957 |
-
|
| 958 |
-
"size": 12,
|
| 959 |
-
"color": "#db2777",
|
| 960 |
-
"line": {"width": 1.5, "color": "#7c3aed"},
|
| 961 |
-
},
|
| 962 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 963 |
)
|
| 964 |
|
| 965 |
-
|
| 966 |
-
|
| 967 |
-
|
| 968 |
-
|
| 969 |
-
|
| 970 |
-
|
| 971 |
-
|
| 972 |
-
|
| 973 |
-
|
| 974 |
-
|
| 975 |
-
|
| 976 |
-
|
| 977 |
-
|
| 978 |
-
"
|
| 979 |
-
|
| 980 |
-
|
| 981 |
-
|
| 982 |
-
|
| 983 |
-
|
| 984 |
-
|
| 985 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 986 |
)
|
| 987 |
-
|
| 988 |
-
|
| 989 |
-
|
| 990 |
-
|
| 991 |
-
|
| 992 |
-
|
| 993 |
-
|
| 994 |
-
|
| 995 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 996 |
)
|
| 997 |
-
|
| 998 |
-
|
| 999 |
-
|
| 1000 |
-
|
| 1001 |
-
|
| 1002 |
-
|
| 1003 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1004 |
)
|
| 1005 |
-
return fig
|
| 1006 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1007 |
|
| 1008 |
-
|
| 1009 |
-
|
| 1010 |
-
|
| 1011 |
-
|
| 1012 |
-
|
| 1013 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1014 |
]
|
| 1015 |
-
|
| 1016 |
-
|
| 1017 |
-
|
| 1018 |
-
|
| 1019 |
-
|
| 1020 |
-
|
| 1021 |
-
|
| 1022 |
-
|
| 1023 |
-
|
| 1024 |
-
|
| 1025 |
-
|
| 1026 |
-
|
| 1027 |
-
"Model",
|
| 1028 |
-
"Platform",
|
| 1029 |
-
"Endpoint Owner",
|
| 1030 |
-
"Optimized",
|
| 1031 |
-
"URL",
|
| 1032 |
-
"Rank",
|
| 1033 |
-
"Median Generation Time (s)",
|
| 1034 |
-
"Min Generation Time (s)",
|
| 1035 |
-
"Price / Image (USD)",
|
| 1036 |
-
"Evaluation Date (UTC)",
|
| 1037 |
-
"Date",
|
| 1038 |
-
"Raw Win Rate",
|
| 1039 |
-
"OneIG Overall Score",
|
| 1040 |
-
}
|
| 1041 |
|
| 1042 |
-
|
| 1043 |
-
|
| 1044 |
-
|
| 1045 |
-
|
| 1046 |
-
|
| 1047 |
-
|
| 1048 |
-
|
| 1049 |
-
|
| 1050 |
-
|
| 1051 |
-
|
| 1052 |
-
|
| 1053 |
-
|
| 1054 |
-
|
| 1055 |
-
|
| 1056 |
-
time_figures = []
|
| 1057 |
-
for plot_column in pareto_columns:
|
| 1058 |
-
if price_column in data.columns:
|
| 1059 |
-
price_fig = _build_pareto_figure(
|
| 1060 |
-
data,
|
| 1061 |
-
plot_column,
|
| 1062 |
-
x_column=price_column,
|
| 1063 |
-
x_title="Price per image (USD)",
|
| 1064 |
-
x_hover_prefix="$",
|
| 1065 |
-
)
|
| 1066 |
-
if price_fig is not None:
|
| 1067 |
-
price_figures.append((plot_column, price_fig))
|
| 1068 |
-
if time_column in data.columns:
|
| 1069 |
-
time_fig = _build_pareto_figure(
|
| 1070 |
-
data,
|
| 1071 |
-
plot_column,
|
| 1072 |
-
x_column=time_column,
|
| 1073 |
-
x_title="Min generation time (s)",
|
| 1074 |
-
x_hover_suffix="s",
|
| 1075 |
-
)
|
| 1076 |
-
if time_fig is not None:
|
| 1077 |
-
time_figures.append((plot_column, time_fig))
|
| 1078 |
-
|
| 1079 |
-
if price_figures or time_figures:
|
| 1080 |
-
gr.Markdown(
|
| 1081 |
-
"### Pareto frontiers\n\n"
|
| 1082 |
-
"<span style='color:#4c1d95;font-size:0.95rem;'>"
|
| 1083 |
-
"Pink = on the frontier (lower cost or time at the same or better score). "
|
| 1084 |
-
"Light purple = below the frontier."
|
| 1085 |
-
"</span>"
|
| 1086 |
)
|
| 1087 |
-
with gr.Row(equal_height=False):
|
| 1088 |
-
with gr.Column(scale=1, min_width=320):
|
| 1089 |
-
gr.Markdown("#### Price vs score")
|
| 1090 |
-
if not price_figures:
|
| 1091 |
-
gr.Markdown("_No price data available._")
|
| 1092 |
-
for plot_column, pareto_fig in price_figures:
|
| 1093 |
-
gr.Markdown(f"**{_display_label(plot_column)}**")
|
| 1094 |
-
gr.Plot(
|
| 1095 |
-
value=pareto_fig,
|
| 1096 |
-
show_label=False,
|
| 1097 |
-
elem_classes="pareto-plot",
|
| 1098 |
-
)
|
| 1099 |
-
with gr.Column(scale=1, min_width=320):
|
| 1100 |
-
gr.Markdown("#### Min generation time vs score")
|
| 1101 |
-
if not time_figures:
|
| 1102 |
-
gr.Markdown("_No min generation time data available._")
|
| 1103 |
-
for plot_column, pareto_fig in time_figures:
|
| 1104 |
-
gr.Markdown(f"**{_display_label(plot_column)}**")
|
| 1105 |
-
gr.Plot(
|
| 1106 |
-
value=pareto_fig,
|
| 1107 |
-
show_label=False,
|
| 1108 |
-
elem_classes="pareto-plot",
|
| 1109 |
-
)
|
| 1110 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1111 |
|
| 1112 |
def render_about():
|
| 1113 |
-
with gr.Row():
|
| 1114 |
-
with gr.Column():
|
| 1115 |
-
gr.Markdown(ABOUT_OVERVIEW_CONTENT)
|
| 1116 |
-
with gr.Column():
|
| 1117 |
-
gr.Markdown(ABOUT_DETAILS_CONTENT)
|
| 1118 |
|
| 1119 |
|
| 1120 |
def render_footer():
|
| 1121 |
-
gr.HTML(
|
| 1122 |
-
with gr.Accordion(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1123 |
gr.Markdown(CITATION_CONTENT)
|
|
|
|
| 1 |
from html import escape
|
| 2 |
+
from pathlib import Path
|
| 3 |
+
import base64
|
| 4 |
import random
|
| 5 |
|
| 6 |
import gradio as gr
|
| 7 |
import pandas as pd
|
| 8 |
import plotly.graph_objects as go
|
| 9 |
|
| 10 |
+
from model_display import display_model_name
|
| 11 |
+
|
| 12 |
+
_LOGO_PATH = Path(__file__).resolve().parent / "pruna-logo.png"
|
| 13 |
+
_LOGO_DATA_URI = (
|
| 14 |
+
"data:image/png;base64,"
|
| 15 |
+
+ base64.b64encode(_LOGO_PATH.read_bytes()).decode("ascii")
|
| 16 |
+
if _LOGO_PATH.exists()
|
| 17 |
+
else "https://playground.pruna.ai/logo.svg"
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
|
|
|
|
| 21 |
MAX_COMPARE_MODELS = 4
|
| 22 |
DEFAULT_COMPARE_PROMPTS = 3
|
| 23 |
MAX_COMPARE_PROMPTS = 8
|
| 24 |
+
MAX_PARETO_METRICS = 8
|
| 25 |
+
_PARETO_SLOT_COUNT = 1 + MAX_PARETO_METRICS * 8
|
| 26 |
+
_PARETO_PRICE_COLUMN = "Price / Image (USD)"
|
| 27 |
+
_PARETO_TIME_COLUMN = "Min Generation Time (s)"
|
| 28 |
+
|
| 29 |
+
TAB_LEADERBOARDS = "leaderboards"
|
| 30 |
+
TAB_PARETO = "pareto"
|
| 31 |
+
TAB_SAMPLES = "samples"
|
| 32 |
+
TAB_ABOUT = "about"
|
| 33 |
+
|
| 34 |
+
_MODEL_CHOICES_CACHE = {}
|
| 35 |
+
_VIEW_EVENTS = {
|
| 36 |
+
"show_progress": "hidden",
|
| 37 |
+
"trigger_mode": "always_last",
|
| 38 |
+
"concurrency_id": "workspace-views",
|
| 39 |
+
"concurrency_limit": 1,
|
| 40 |
+
}
|
| 41 |
|
| 42 |
+
ABOUT_OVERVIEW_CONTENT = """
|
| 43 |
+
# About P-Bench
|
| 44 |
+
|
| 45 |
+
P-Bench compares **text-to-image models**, including optimized or accelerated
|
| 46 |
+
endpoints, on **quality, speed, and price**. Each view is a **dataset** scored
|
| 47 |
+
with a **metric**, written as `Dataset | Metric`. There is no single score
|
| 48 |
+
across P-Bench.
|
| 49 |
+
|
| 50 |
+
## How to read it
|
| 51 |
+
|
| 52 |
+
1. Pick a **dataset** and a **metric**.
|
| 53 |
+
2. **Leaderboards**: ranked by that metric. Price and generation time sit in
|
| 54 |
+
the same table when the source publishes them.
|
| 55 |
+
3. **Pareto plots**: mark models that are not beaten on both higher score
|
| 56 |
+
and lower price (or time). Only datasets with price or generation time
|
| 57 |
+
can open this tab (not Arena AI).
|
| 58 |
+
4. **Samples**: the same prompts, side by side. Only for datasets we
|
| 59 |
+
generated (Qwen Image Dataset and OneIG Alignment Dataset).
|
| 60 |
+
|
| 61 |
+
## How a score is made
|
| 62 |
+
|
| 63 |
+
On **Qwen Image Dataset** and **OneIG Alignment Dataset**:
|
| 64 |
+
|
| 65 |
+
1. Each endpoint is given the same prompt suite.
|
| 66 |
+
2. It generates one image per prompt when the run succeeds. Not every model
|
| 67 |
+
has every prompt or every metric.
|
| 68 |
+
3. Quality is scored automatically (OneIG alignment, P-Judger) and, where
|
| 69 |
+
available, by human preference (Datapoint Elo, Rapidata Elo).
|
| 70 |
+
4. Price per image and generation time are joined from the evaluation table.
|
| 71 |
+
|
| 72 |
+
**Artificial Analysis** and **Arena AI** are external leaderboards. We import
|
| 73 |
+
their published Elos (and Artificial Analysis price). We do not run their
|
| 74 |
+
prompt suites, so samples are not shown.
|
| 75 |
+
|
| 76 |
+
## Current datasets
|
| 77 |
+
|
| 78 |
+
### Qwen Image Dataset
|
| 79 |
+
100 prompts from the 1,000-prompt Qwen Image Bench set, sampled for coverage
|
| 80 |
+
across its fine-grained (L3) categories. Metrics include Datapoint Elo,
|
| 81 |
+
Rapidata Elo, and P-Judger. Samples are available.
|
| 82 |
+
|
| 83 |
+
### OneIG Alignment Dataset
|
| 84 |
+
Prompt-image **alignment** on anime / stylization (100), human / portrait
|
| 85 |
+
(100), and general object prompts (99). This is the alignment slice of
|
| 86 |
+
OneIG, not the full suite. Alignment Overall is the mean of the category
|
| 87 |
+
scores that exist for that row. Also includes Datapoint category Elo,
|
| 88 |
+
Rapidata Elo, and P-Judger. Samples are available.
|
| 89 |
+
|
| 90 |
+
### Artificial Analysis Dataset
|
| 91 |
+
External text-to-image Elo and price per image from Artificial Analysis.
|
| 92 |
+
Their prompt set is private, so samples are not shown. Pareto plots use
|
| 93 |
+
price vs score only.
|
| 94 |
+
|
| 95 |
+
### Arena AI Dataset
|
| 96 |
+
External text-to-image Elo (overall and category) from Arena AI. Their
|
| 97 |
+
prompt set is private, so samples are not shown. Price and generation time
|
| 98 |
+
are not in this export, so Pareto plots are unavailable.
|
| 99 |
"""
|
| 100 |
|
| 101 |
+
ABOUT_DETAILS_CONTENT = """
|
| 102 |
+
# Metrics & limits
|
| 103 |
+
|
| 104 |
+
## Metrics
|
| 105 |
+
|
| 106 |
+
- **Alignment**: how well the image matches the prompt (OneIG category scores).
|
| 107 |
+
- **P-Judger**: Pruna's automatic judger. A private scoring model that rates
|
| 108 |
+
each (prompt, image) pair in one pass and returns an absolute quality score,
|
| 109 |
+
not a pairwise Elo. It is not a text-to-image model on the leaderboard. Use
|
| 110 |
+
it alongside Datapoint and Rapidata Elo, not instead of them.
|
| 111 |
+
- **Datapoint Elo**: human-preference Elo from Datapoint pairwise comparisons.
|
| 112 |
+
- **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.
|
| 113 |
+
Rapidata rejects prompts over 400 characters, so this Elo is on a subset
|
| 114 |
+
of each suite (see Setup). Rapidata is not a dataset.
|
| 115 |
+
- **Artificial Analysis Elo**: Elo published by Artificial Analysis on their
|
| 116 |
+
own dataset.
|
| 117 |
+
- **Arena Elo**: Elo published by Arena AI on their own dataset, plus
|
| 118 |
+
category Elos (branding, 3D, cartoon/anime, photorealistic, art, portraits,
|
| 119 |
+
text rendering).
|
| 120 |
+
- **Generation time**: median and minimum generation time in seconds, as
|
| 121 |
+
reported in the evaluation table. This is not a p95, and we do not state
|
| 122 |
+
warm vs cold or concurrent load. Not available for Arena AI.
|
| 123 |
+
- **Price**: USD per image in the evaluation table. We do not state list
|
| 124 |
+
price vs amount paid, or whether failed generations are included. Not
|
| 125 |
+
available for Arena AI.
|
| 126 |
+
|
| 127 |
+
Scores from different datasets or metrics are **not interchangeable**. A high
|
| 128 |
+
OneIG alignment score is not the same quantity as a Datapoint Elo. Compare
|
| 129 |
+
models *within* a Dataset | Metric view.
|
| 130 |
+
|
| 131 |
+
## Setup
|
| 132 |
+
|
| 133 |
+
- **Evaluation window:** July–August 2026.
|
| 134 |
+
- **Update policy:** numbers come from evaluation snapshots in the tables,
|
| 135 |
+
not a live API poll.
|
| 136 |
+
- **Prompt counts:** OneIG Alignment uses 100 anime, 100 human, and 99 object
|
| 137 |
+
prompts (299 total). Qwen Image Dataset uses 100 prompts sampled from the
|
| 138 |
+
1,000-prompt pool for roughly even coverage of its fine-grained (L3)
|
| 139 |
+
categories. Artificial Analysis and Arena AI use their own private prompt
|
| 140 |
+
sets.
|
| 141 |
+
- **Generation (Qwen and OneIG):** one image per prompt per endpoint when
|
| 142 |
+
the run exists. Default resolution is 1024×1024. Exceptions: FLUX 1.1 Pro
|
| 143 |
+
Ultra at 2K, FLUX 2 Flex at 1008×1008, and any endpoint labeled 2K. The
|
| 144 |
+
seed is derived from the prompt, so every model gets the same seed for the
|
| 145 |
+
same prompt. Steps, CFG, prompt rewrite, and safety filters follow each
|
| 146 |
+
endpoint's default. This does not describe Artificial Analysis or Arena AI.
|
| 147 |
+
- **Datapoint (Qwen and OneIG):** every model pair is compared on every
|
| 148 |
+
prompt, with 10 votes per battle.
|
| 149 |
+
- **Rapidata (Qwen and OneIG):** prompts longer than 400 characters are
|
| 150 |
+
dropped, leaving 212 OneIG prompts and 85 Qwen Image Dataset prompts. 4
|
| 151 |
+
votes per pair; about 26,000 votes on OneIG and 35,000 on Qwen Image
|
| 152 |
+
Dataset.
|
| 153 |
+
|
| 154 |
+
## Limits
|
| 155 |
+
|
| 156 |
+
- Empty cells mean that track was not run or not reported for that model.
|
| 157 |
+
- Artificial Analysis and Arena AI samples, prompts, and (for Arena) price
|
| 158 |
+
or latency are not available to P-Bench.
|
| 159 |
+
- Rapidata Elo is not on the full prompt suite, so it is not directly
|
| 160 |
+
comparable to Datapoint Elo even on the same dataset.
|
| 161 |
+
- Elo ratings can shift when the comparison pool changes: treat them as
|
| 162 |
+
relative rankings for the snapshot, not absolute constants.
|
| 163 |
+
- Close scores can be a tie in practice; the table does not show confidence
|
| 164 |
+
intervals.
|
| 165 |
+
- Optimized or accelerated endpoints (when labeled) may differ from the base
|
| 166 |
+
model publisher's default serving stack.
|
| 167 |
+
- APIs and prices drift; a rank is for the snapshot, not a guarantee.
|
| 168 |
+
|
| 169 |
+
Built by [Pruna AI](https://www.pruna.ai/). If you have questions, reach out
|
| 170 |
+
to us on [Discord](https://discord.gg/JFQmtFKCjd).
|
| 171 |
"""
|
| 172 |
|
| 173 |
+
FOOTER_CONTENT = """
|
| 174 |
+
<div class="page-footer">
|
| 175 |
+
<h3>Join the Pruna AI community</h3>
|
| 176 |
+
<div class="community-footer-links">
|
| 177 |
+
<a rel="nofollow" href="https://x.com/PrunaAI" target="_blank">X</a>
|
| 178 |
+
<a rel="nofollow" href="https://www.linkedin.com/company/pruna-ai" target="_blank">LinkedIn</a>
|
| 179 |
+
<a rel="nofollow" href="https://www.instagram.com/pruna.ai/" target="_blank">Instagram</a>
|
| 180 |
+
<a rel="nofollow" href="https://discord.gg/JFQmtFKCjd" target="_blank">Discord</a>
|
| 181 |
+
<a rel="nofollow" href="https://github.com/PrunaAI/pruna" target="_blank">GitHub</a>
|
| 182 |
+
<a rel="nofollow" href="https://www.pruna.ai/" target="_blank">pruna.ai</a>
|
| 183 |
+
</div>
|
| 184 |
+
</div>
|
| 185 |
"""
|
| 186 |
|
| 187 |
+
CITATION_CONTENT = """
|
| 188 |
+
```bibtex
|
| 189 |
+
@misc{P-Bench,
|
| 190 |
+
title={P-Bench: A Leaderboard for Text-to-Image Models},
|
| 191 |
+
author={PrunaAI},
|
| 192 |
+
year={2026},
|
| 193 |
+
howpublished={\\url{https://huggingface.co/spaces/PrunaAI/P-Bench}}
|
| 194 |
+
}
|
| 195 |
+
```
|
| 196 |
"""
|
| 197 |
|
|
|
|
|
|
|
| 198 |
|
| 199 |
+
def render_header():
|
| 200 |
+
gr.HTML(
|
| 201 |
+
f"""
|
| 202 |
+
<header class="app-header">
|
| 203 |
+
<div class="app-header-bar">
|
| 204 |
+
<div class="app-header-brand">
|
| 205 |
+
<img class="app-header-logo" src="{_LOGO_DATA_URI}" alt="" />
|
| 206 |
+
<h1>P-Bench</h1>
|
| 207 |
+
</div>
|
| 208 |
+
<button type="button" class="theme-toggle" data-mode="dark" aria-label="Switch to light mode" title="Switch to light mode">
|
| 209 |
+
<svg class="theme-icon-sun" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">
|
| 210 |
+
<circle cx="12" cy="12" r="4"></circle>
|
| 211 |
+
<path d="M12 2v2M12 20v2M4.93 4.93l1.41 1.41M17.66 17.66l1.41 1.41M2 12h2M20 12h2M4.93 19.07l1.41-1.41M17.66 6.34l1.41-1.41"></path>
|
| 212 |
+
</svg>
|
| 213 |
+
<svg class="theme-icon-moon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">
|
| 214 |
+
<path d="M21 14.5A8.5 8.5 0 1 1 9.5 3 7 7 0 0 0 21 14.5z"></path>
|
| 215 |
+
</svg>
|
| 216 |
+
</button>
|
| 217 |
+
</div>
|
| 218 |
+
<p class="app-header-tagline">Compare text-to-image models on quality, speed, and price</p>
|
| 219 |
+
</header>
|
| 220 |
+
""",
|
| 221 |
+
padding=False,
|
| 222 |
+
elem_classes="app-header-host",
|
| 223 |
+
)
|
| 224 |
|
|
|
|
|
|
|
| 225 |
|
| 226 |
+
def _item(items, item_id):
|
| 227 |
+
for item in items:
|
| 228 |
+
if item["id"] == item_id:
|
| 229 |
+
return item
|
| 230 |
+
return items[0] if items else None
|
| 231 |
|
|
|
|
|
|
|
| 232 |
|
| 233 |
+
def _dataset_choices(datasets, *, require_samples=False, require_pareto=False):
|
| 234 |
+
return [
|
| 235 |
+
(dataset["name"], dataset["id"])
|
| 236 |
+
for dataset in datasets
|
| 237 |
+
if (not require_samples or dataset.get("samples"))
|
| 238 |
+
and (not require_pareto or _dataset_has_pareto(datasets, dataset["id"]))
|
| 239 |
+
]
|
| 240 |
|
|
|
|
|
|
|
|
|
|
| 241 |
|
| 242 |
+
def _dataset_has_samples(datasets, dataset_id):
|
| 243 |
+
dataset = _item(datasets, dataset_id)
|
| 244 |
+
return bool(dataset and dataset.get("samples"))
|
| 245 |
|
|
|
|
|
|
|
|
|
|
| 246 |
|
| 247 |
+
def _dataset_has_pareto(datasets, dataset_id):
|
| 248 |
+
dataset = _item(datasets, dataset_id)
|
| 249 |
+
columns = getattr(dataset.get("data") if dataset else None, "columns", [])
|
| 250 |
+
return _PARETO_PRICE_COLUMN in columns or _PARETO_TIME_COLUMN in columns
|
| 251 |
|
|
|
|
|
|
|
|
|
|
| 252 |
|
| 253 |
+
def _dataset_dropdown_update(datasets, tab, dataset_id):
|
| 254 |
+
"""Limit the dataset list to what the current tab can show."""
|
| 255 |
+
return gr.update(
|
| 256 |
+
choices=_dataset_choices(
|
| 257 |
+
datasets,
|
| 258 |
+
require_samples=tab == TAB_SAMPLES
|
| 259 |
+
and _dataset_has_samples(datasets, dataset_id),
|
| 260 |
+
require_pareto=tab == TAB_PARETO
|
| 261 |
+
and _dataset_has_pareto(datasets, dataset_id),
|
| 262 |
+
),
|
| 263 |
+
value=dataset_id,
|
| 264 |
+
)
|
| 265 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 266 |
|
| 267 |
+
ALL_METRICS_ID = "__all__"
|
| 268 |
|
|
|
|
|
|
|
| 269 |
|
| 270 |
+
def _metric_choices(datasets, metrics, dataset_id):
|
| 271 |
+
dataset = _item(datasets, dataset_id)
|
| 272 |
+
if not dataset:
|
| 273 |
+
return []
|
| 274 |
+
allowed = set(dataset.get("metric_ids") or [])
|
| 275 |
+
data = dataset.get("data")
|
| 276 |
+
columns = getattr(data, "columns", [])
|
| 277 |
+
return [
|
| 278 |
+
(_display_label(metric["column"]), metric["id"])
|
| 279 |
+
for metric in metrics
|
| 280 |
+
if metric["id"] in allowed and metric["column"] in columns
|
| 281 |
+
]
|
| 282 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 283 |
|
| 284 |
+
def _metric_dropdown_choices(datasets, metrics, dataset_id):
|
| 285 |
+
choices = _metric_choices(datasets, metrics, dataset_id)
|
| 286 |
+
if not choices:
|
| 287 |
+
return []
|
| 288 |
+
return [("Select all", ALL_METRICS_ID)] + choices
|
| 289 |
|
|
|
|
|
|
|
|
|
|
| 290 |
|
| 291 |
+
def _normalize_metric_ids(metric_id):
|
| 292 |
+
if metric_id is None or metric_id == "":
|
| 293 |
+
return []
|
| 294 |
+
if isinstance(metric_id, (list, tuple)):
|
| 295 |
+
return [item for item in metric_id if item]
|
| 296 |
+
return [metric_id]
|
| 297 |
|
|
|
|
| 298 |
|
| 299 |
+
def _coerce_metric(datasets, metrics, dataset_id, metric_id):
|
| 300 |
+
"""Valid metric ids for this dataset. Empty means all metrics."""
|
| 301 |
+
wanted = _normalize_metric_ids(metric_id)
|
| 302 |
+
valid_ids = [choice[1] for choice in _metric_choices(datasets, metrics, dataset_id)]
|
| 303 |
+
valid = set(valid_ids)
|
| 304 |
+
if ALL_METRICS_ID in wanted:
|
| 305 |
+
return list(valid_ids)
|
| 306 |
+
return [item for item in wanted if item in valid]
|
| 307 |
|
|
|
|
|
|
|
|
|
|
| 308 |
|
| 309 |
+
def _metric_dropdown_value(metric_id):
|
| 310 |
+
return [
|
| 311 |
+
item
|
| 312 |
+
for item in _normalize_metric_ids(metric_id)
|
| 313 |
+
if item != ALL_METRICS_ID
|
| 314 |
+
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 316 |
|
| 317 |
+
def _model_choices(datasets, dataset_id):
|
| 318 |
+
cached = _MODEL_CHOICES_CACHE.get(dataset_id)
|
| 319 |
+
if cached is not None:
|
| 320 |
+
return cached
|
| 321 |
+
dataset = _item(datasets, dataset_id)
|
| 322 |
+
data = dataset.get("data") if dataset else None
|
| 323 |
+
if data is None or "Model" not in getattr(data, "columns", []):
|
| 324 |
+
_MODEL_CHOICES_CACHE[dataset_id] = []
|
| 325 |
+
return []
|
| 326 |
+
models = data["Model"].dropna().astype(str).unique().tolist()
|
| 327 |
+
# (label, value) so the UI shows the shared name but filters on the raw id.
|
| 328 |
+
choices = sorted(
|
| 329 |
+
((display_model_name(model), model) for model in models),
|
| 330 |
+
key=lambda item: item[0].casefold(),
|
| 331 |
+
)
|
| 332 |
+
_MODEL_CHOICES_CACHE[dataset_id] = choices
|
| 333 |
+
return choices
|
| 334 |
|
| 335 |
|
| 336 |
+
def _model_choice_values(choices):
|
| 337 |
+
return [choice[1] if isinstance(choice, tuple) else choice for choice in choices]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 338 |
|
| 339 |
|
| 340 |
+
def _metric_columns(datasets, metrics, dataset_id):
|
| 341 |
+
dataset = _item(datasets, dataset_id)
|
| 342 |
+
if not dataset:
|
| 343 |
return []
|
| 344 |
+
data = dataset.get("data")
|
| 345 |
+
columns = getattr(data, "columns", [])
|
| 346 |
+
allowed = set(dataset.get("metric_ids") or [])
|
|
|
|
|
|
|
|
|
|
|
|
|
| 347 |
return [
|
| 348 |
+
metric["column"]
|
| 349 |
+
for metric in metrics
|
| 350 |
+
if metric["id"] in allowed and metric["column"] in columns
|
| 351 |
]
|
| 352 |
|
| 353 |
|
| 354 |
+
_LEADERBOARD_IDENTITY_COLUMNS = [
|
| 355 |
+
"Model",
|
| 356 |
+
"Platform",
|
| 357 |
+
"Endpoint Owner",
|
| 358 |
+
"Optimized",
|
| 359 |
+
]
|
| 360 |
+
_LEADERBOARD_META_COLUMNS = [
|
| 361 |
+
"Median Generation Time (s)",
|
| 362 |
+
"Min Generation Time (s)",
|
| 363 |
+
"Price / Image (USD)",
|
| 364 |
+
"Evaluation Date (UTC)",
|
| 365 |
+
"Date",
|
| 366 |
+
]
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
def _columns_for_metric(dataset, score_columns):
|
| 370 |
+
"""When metrics are selected, show identity + those scores + time/price."""
|
| 371 |
+
available = list(getattr(dataset.get("data"), "columns", [])) or list(
|
| 372 |
+
dataset.get("columns") or []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 373 |
)
|
| 374 |
+
identity = [column for column in _LEADERBOARD_IDENTITY_COLUMNS if column in available]
|
| 375 |
+
meta = [column for column in _LEADERBOARD_META_COLUMNS if column in available]
|
| 376 |
+
scores = [column for column in (score_columns or []) if column in available]
|
| 377 |
+
if scores:
|
| 378 |
+
return [*identity, *scores, *meta]
|
| 379 |
+
return [column for column in (dataset.get("columns") or available) if column != "URL"]
|
| 380 |
|
| 381 |
+
|
| 382 |
+
def resolve_view(datasets, metrics, dataset_id, metric_id):
|
| 383 |
+
dataset = _item(datasets, dataset_id)
|
| 384 |
+
if not dataset:
|
| 385 |
+
return None
|
| 386 |
+
data_columns = getattr(dataset.get("data"), "columns", [])
|
| 387 |
+
score_columns = []
|
| 388 |
+
for metric_key in _coerce_metric(datasets, metrics, dataset_id, metric_id):
|
| 389 |
+
metric = _item(metrics, metric_key)
|
| 390 |
+
if metric and metric["column"] in data_columns:
|
| 391 |
+
score_columns.append(metric["column"])
|
| 392 |
+
if score_columns:
|
| 393 |
+
columns = _columns_for_metric(dataset, score_columns)
|
| 394 |
+
else:
|
| 395 |
+
columns = [
|
| 396 |
+
column
|
| 397 |
+
for column in (dataset.get("columns") or [])
|
| 398 |
+
if column != "URL"
|
| 399 |
]
|
| 400 |
+
score_columns = _metric_columns(datasets, metrics, dataset_id)
|
| 401 |
+
return {
|
| 402 |
+
"data": dataset["data"],
|
| 403 |
+
"columns": columns,
|
| 404 |
+
"score_column": score_columns[0] if score_columns else None,
|
| 405 |
+
"score_columns": score_columns,
|
| 406 |
+
"samples": dataset.get("samples"),
|
| 407 |
+
"note": dataset.get("note"),
|
| 408 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
|
| 410 |
|
| 411 |
def _format_leaderboard_cell(column, value):
|
| 412 |
if pd.isna(value) or value is None or value == "":
|
| 413 |
+
return "-"
|
| 414 |
label = str(column).lower()
|
| 415 |
if label == "rank":
|
| 416 |
return str(int(value))
|
|
|
|
| 422 |
except (TypeError, ValueError):
|
| 423 |
return escape(str(value))
|
| 424 |
if label in {"model", "platform", "endpoint owner", "optimized"}:
|
| 425 |
+
if label == "model":
|
| 426 |
+
return escape(display_model_name(value))
|
| 427 |
return escape(str(value))
|
| 428 |
try:
|
| 429 |
number = float(value)
|
|
|
|
| 454 |
return "number"
|
| 455 |
|
| 456 |
|
| 457 |
+
def _leaderboard_cell_class(column):
|
| 458 |
+
if column == "Rank":
|
| 459 |
+
return "rank"
|
| 460 |
+
if column == "Model":
|
| 461 |
+
return "model-cell"
|
| 462 |
+
if _leaderboard_sort_type(column) == "text":
|
| 463 |
+
return "text-cell"
|
| 464 |
+
return "metric-score"
|
| 465 |
+
|
| 466 |
+
|
| 467 |
+
def _assign_leaderboard_ranks(data, overall_column):
|
| 468 |
+
"""Rank the full table by the selected metric. Filters keep these numbers."""
|
| 469 |
+
if data is None:
|
| 470 |
+
return data
|
| 471 |
+
ranked = data.copy()
|
| 472 |
+
if "Rank" in ranked.columns:
|
| 473 |
+
ranked = ranked.drop(columns=["Rank"])
|
| 474 |
+
if overall_column and overall_column in ranked.columns:
|
| 475 |
+
ranked = ranked.sort_values(
|
| 476 |
+
overall_column, ascending=False, na_position="last"
|
| 477 |
+
)
|
| 478 |
+
ranked = ranked.reset_index(drop=True)
|
| 479 |
+
ranked.insert(0, "Rank", ranked.index + 1)
|
| 480 |
+
return ranked
|
| 481 |
+
|
| 482 |
+
|
| 483 |
+
def _leaderboard_html(data, columns):
|
| 484 |
+
leaderboard = _leaderboard_dataframe(data, columns)
|
| 485 |
if leaderboard.empty:
|
| 486 |
return (
|
| 487 |
'<div class="ranking-table-scroll">'
|
|
|
|
| 492 |
header_cells = []
|
| 493 |
for index, column in enumerate(leaderboard.columns):
|
| 494 |
sort_type = _leaderboard_sort_type(column)
|
| 495 |
+
cell_class = _leaderboard_cell_class(column)
|
| 496 |
header_cells.append(
|
| 497 |
+
f'<th class="sortable-col {cell_class}" data-col="{index}" '
|
| 498 |
+
f'data-type="{sort_type}" title="Sort by {escape(str(column))}">'
|
| 499 |
+
f"{escape(str(column))}</th>"
|
| 500 |
)
|
| 501 |
body_rows = []
|
| 502 |
for _, row in leaderboard.iterrows():
|
| 503 |
cells = []
|
| 504 |
for column in leaderboard.columns:
|
| 505 |
+
css = _leaderboard_cell_class(column)
|
| 506 |
sort_value = escape(_leaderboard_sort_value(column, row[column]), quote=True)
|
| 507 |
if column == "Model":
|
|
|
|
| 508 |
cells.append(
|
| 509 |
f'<td class="{css}" data-sort-value="{sort_value}">'
|
| 510 |
f"<strong>{_format_leaderboard_cell(column, row[column])}</strong></td>"
|
|
|
|
| 528 |
"""
|
| 529 |
|
| 530 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 531 |
def _filter_choices(data, column):
|
| 532 |
+
if data is None or column not in data.columns:
|
| 533 |
return []
|
| 534 |
return sorted(data[column].dropna().astype(str).unique().tolist())
|
| 535 |
|
| 536 |
|
| 537 |
+
def _filter_leaderboard(data, platform, owner, optimized, models=None):
|
| 538 |
filtered = data.copy()
|
| 539 |
+
if models:
|
| 540 |
+
if "Model" in filtered.columns:
|
| 541 |
+
filtered = filtered[filtered["Model"].astype(str).isin(models)]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 542 |
|
| 543 |
for column, values in [
|
| 544 |
("Platform", platform),
|
|
|
|
| 550 |
return filtered
|
| 551 |
|
| 552 |
|
| 553 |
+
def _leaderboard_dataframe(data, columns):
|
| 554 |
+
ordered = []
|
| 555 |
+
if "Rank" in data.columns:
|
| 556 |
+
ordered.append("Rank")
|
| 557 |
+
for column in columns:
|
| 558 |
+
if (
|
| 559 |
+
column in data.columns
|
| 560 |
+
and column not in {"URL", "Rank"}
|
| 561 |
+
and column not in ordered
|
| 562 |
+
):
|
| 563 |
+
ordered.append(column)
|
| 564 |
+
return data[ordered].rename(columns=_display_label)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 565 |
|
| 566 |
|
| 567 |
def _display_label(column):
|
| 568 |
labels = {
|
|
|
|
| 569 |
"OneIG Overall Score": "Overall",
|
| 570 |
"OneIG (Anime Alignment)": "Anime",
|
| 571 |
"OneIG (Human Alignment)": "Human",
|
|
|
|
| 573 |
"OneIG Anime Elo": "Anime Elo (Datapoint)",
|
| 574 |
"OneIG Human Elo": "Human Elo (Datapoint)",
|
| 575 |
"OneIG Object Elo": "Object Elo (Datapoint)",
|
| 576 |
+
"P-Judge Overall": "P-Judger (Pruna)",
|
| 577 |
"Datapoint Elo": "Datapoint Elo",
|
| 578 |
"Rapidata Elo": "Rapidata Elo",
|
| 579 |
+
"Artificial Analysis Elo": "Artificial Analysis Elo",
|
| 580 |
+
"Arena Elo": "Overall Elo",
|
| 581 |
+
"Arena Branding / Commercial Elo": "Branding / Commercial",
|
| 582 |
+
"Arena 3D Imaging Elo": "3D Imaging",
|
| 583 |
+
"Arena Cartoon / Anime Elo": "Cartoon / Anime",
|
| 584 |
+
"Arena Photorealistic Elo": "Photorealistic",
|
| 585 |
+
"Arena Art Elo": "Art",
|
| 586 |
+
"Arena Portraits Elo": "Portraits",
|
| 587 |
+
"Arena Text Rendering Elo": "Text Rendering",
|
| 588 |
"Raw Win Rate": "Raw win rate",
|
| 589 |
"Median Generation Time (s)": "Median generation time",
|
| 590 |
"Min Generation Time (s)": "Min generation time",
|
|
|
|
| 595 |
return labels.get(column, column)
|
| 596 |
|
| 597 |
|
| 598 |
+
def _format_price(value):
|
| 599 |
+
return "-" if pd.isna(value) or value is None else f"${float(value):.3f}"
|
| 600 |
|
| 601 |
|
| 602 |
+
def _pareto_frontier_mask(x_values, scores):
|
| 603 |
+
"""True for non-dominated points when maximizing score and minimizing x."""
|
| 604 |
+
n = len(x_values)
|
| 605 |
+
mask = [True] * n
|
| 606 |
+
for i in range(n):
|
| 607 |
+
for j in range(n):
|
| 608 |
+
if i == j:
|
| 609 |
+
continue
|
| 610 |
+
better_or_equal = x_values[j] <= x_values[i] and scores[j] >= scores[i]
|
| 611 |
+
strictly_better = x_values[j] < x_values[i] or scores[j] > scores[i]
|
| 612 |
+
if better_or_equal and strictly_better:
|
| 613 |
+
mask[i] = False
|
| 614 |
+
break
|
| 615 |
+
return mask
|
| 616 |
|
| 617 |
|
| 618 |
+
def _pareto_unavailable_html(message):
|
| 619 |
+
return (
|
| 620 |
+
"<p class='pareto-note-copy'>"
|
| 621 |
+
f"{escape(message)}"
|
| 622 |
+
"</p>"
|
| 623 |
+
)
|
| 624 |
|
| 625 |
|
| 626 |
+
def _pareto_note_update(message):
|
| 627 |
+
if message:
|
| 628 |
+
return gr.update(
|
| 629 |
+
value=_pareto_unavailable_html(message),
|
| 630 |
+
visible=True,
|
| 631 |
+
)
|
| 632 |
+
return gr.update(value="", visible=False)
|
| 633 |
+
|
| 634 |
+
|
| 635 |
+
def _pareto_plot_update(fig):
|
| 636 |
+
if fig is not None:
|
| 637 |
+
return gr.update(value=fig, visible=True)
|
| 638 |
+
return gr.update(value=None, visible=False)
|
| 639 |
+
|
| 640 |
+
|
| 641 |
+
def _skip_all(count):
|
| 642 |
+
return tuple(gr.skip() for _ in range(count))
|
| 643 |
+
|
| 644 |
|
| 645 |
+
def _pareto_skip_updates():
|
| 646 |
+
return _skip_all(_PARETO_SLOT_COUNT)
|
| 647 |
+
|
| 648 |
+
|
| 649 |
+
def _selection_key(dataset_id, metric_id, models):
|
| 650 |
+
return (
|
| 651 |
+
dataset_id,
|
| 652 |
+
tuple(_normalize_metric_ids(metric_id)),
|
| 653 |
+
tuple(models or ()),
|
| 654 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 655 |
|
| 656 |
|
| 657 |
+
def _applied_key(view_state):
|
| 658 |
+
view_state = view_state or {}
|
| 659 |
+
return _selection_key(
|
| 660 |
+
view_state.get("dataset_id"),
|
| 661 |
+
view_state.get("metric_id"),
|
| 662 |
+
view_state.get("models"),
|
| 663 |
+
)
|
| 664 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 665 |
|
| 666 |
+
def _build_pareto_figure(
|
| 667 |
+
data,
|
| 668 |
+
score_column,
|
| 669 |
+
x_column,
|
| 670 |
+
x_title,
|
| 671 |
+
x_hover_prefix="",
|
| 672 |
+
x_hover_suffix="",
|
| 673 |
+
):
|
| 674 |
+
scatter = (
|
| 675 |
+
data[["Model", score_column, x_column]]
|
| 676 |
+
.dropna()
|
| 677 |
+
.copy()
|
| 678 |
+
.reset_index(drop=True)
|
| 679 |
+
)
|
| 680 |
+
if scatter.empty:
|
| 681 |
+
return None
|
| 682 |
+
|
| 683 |
+
x_values = scatter[x_column].astype(float).tolist()
|
| 684 |
+
scores = scatter[score_column].astype(float).tolist()
|
| 685 |
+
on_frontier = _pareto_frontier_mask(x_values, scores)
|
| 686 |
|
| 687 |
+
dominated = scatter.loc[[not flag for flag in on_frontier]].copy()
|
| 688 |
+
frontier = scatter.loc[on_frontier].sort_values(x_column).copy()
|
| 689 |
+
if not dominated.empty:
|
| 690 |
+
dominated["Model"] = dominated["Model"].map(display_model_name)
|
| 691 |
+
if not frontier.empty:
|
| 692 |
+
frontier["Model"] = frontier["Model"].map(display_model_name)
|
| 693 |
+
hover = (
|
| 694 |
+
"<b>%{text}</b><br>"
|
| 695 |
+
f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"
|
| 696 |
+
"<br>Score: %{y:.4f}<extra></extra>"
|
| 697 |
)
|
| 698 |
+
|
| 699 |
+
fig = go.Figure()
|
| 700 |
+
if not dominated.empty:
|
| 701 |
+
fig.add_trace(
|
| 702 |
+
go.Scatter(
|
| 703 |
+
x=dominated[x_column],
|
| 704 |
+
y=dominated[score_column],
|
| 705 |
+
mode="markers",
|
| 706 |
+
name="Below frontier",
|
| 707 |
+
text=dominated["Model"],
|
| 708 |
+
hovertemplate=hover,
|
| 709 |
+
marker={
|
| 710 |
+
"size": 9,
|
| 711 |
+
"color": "#d8b4fe",
|
| 712 |
+
"opacity": 0.8,
|
| 713 |
+
"line": {"width": 0},
|
| 714 |
+
},
|
| 715 |
+
)
|
| 716 |
)
|
| 717 |
+
if not frontier.empty:
|
| 718 |
+
fig.add_trace(
|
| 719 |
+
go.Scatter(
|
| 720 |
+
x=frontier[x_column],
|
| 721 |
+
y=frontier[score_column],
|
| 722 |
+
mode="lines+markers",
|
| 723 |
+
name="On frontier",
|
| 724 |
+
text=frontier["Model"],
|
| 725 |
+
hovertemplate=hover,
|
| 726 |
+
line={"color": "#69a45c", "width": 2.5},
|
| 727 |
+
marker={
|
| 728 |
+
"size": 12,
|
| 729 |
+
"color": "#69a45c",
|
| 730 |
+
"line": {"width": 1.5, "color": "#86c077"},
|
| 731 |
+
},
|
| 732 |
+
)
|
| 733 |
)
|
| 734 |
+
|
| 735 |
+
score_label = _display_label(score_column)
|
| 736 |
+
fig.update_layout(
|
| 737 |
+
title=None,
|
| 738 |
+
xaxis_title=x_title,
|
| 739 |
+
yaxis_title=score_label,
|
| 740 |
+
autosize=True,
|
| 741 |
+
height=420,
|
| 742 |
+
margin={"l": 56, "r": 28, "t": 28, "b": 80},
|
| 743 |
+
legend={
|
| 744 |
+
"orientation": "h",
|
| 745 |
+
"yanchor": "top",
|
| 746 |
+
"y": -0.24,
|
| 747 |
+
"xanchor": "center",
|
| 748 |
+
"x": 0.5,
|
| 749 |
+
"bgcolor": "rgba(0,0,0,0)",
|
| 750 |
+
"font": {"color": "#d4d4d4", "size": 12},
|
| 751 |
+
},
|
| 752 |
+
plot_bgcolor="#1d1429",
|
| 753 |
+
paper_bgcolor="#171021",
|
| 754 |
+
font={"color": "#d4d4d4", "size": 13},
|
| 755 |
)
|
| 756 |
+
axis_font = {"color": "#fafafa", "size": 13}
|
| 757 |
+
tick_font = {"color": "#a3a3a3", "size": 12}
|
| 758 |
+
fig.update_xaxes(
|
| 759 |
+
showgrid=True,
|
| 760 |
+
gridcolor="rgba(74, 57, 98, 0.55)",
|
| 761 |
+
zeroline=False,
|
| 762 |
+
title_font=axis_font,
|
| 763 |
+
tickfont=tick_font,
|
| 764 |
+
color="#d4d4d4",
|
| 765 |
+
)
|
| 766 |
+
fig.update_yaxes(
|
| 767 |
+
showgrid=True,
|
| 768 |
+
gridcolor="rgba(74, 57, 98, 0.55)",
|
| 769 |
+
zeroline=False,
|
| 770 |
+
title_font=axis_font,
|
| 771 |
+
tickfont=tick_font,
|
| 772 |
+
color="#d4d4d4",
|
| 773 |
+
)
|
| 774 |
+
return fig
|
| 775 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 776 |
|
| 777 |
+
def _pareto_axis(data, score_column, x_column, x_title, missing_message, empty_message, **hover):
|
| 778 |
+
if x_column not in data.columns:
|
| 779 |
+
return None, missing_message
|
| 780 |
+
fig = _build_pareto_figure(
|
| 781 |
+
data,
|
| 782 |
+
score_column,
|
| 783 |
+
x_column=x_column,
|
| 784 |
+
x_title=x_title,
|
| 785 |
+
**hover,
|
| 786 |
)
|
| 787 |
+
if fig is None:
|
| 788 |
+
return None, empty_message
|
| 789 |
+
return fig, None
|
| 790 |
+
|
| 791 |
+
|
| 792 |
+
def _pareto_pair(data, score_column):
|
| 793 |
+
score_missing = "No score data is available for this metric."
|
| 794 |
+
if data is None or not score_column or score_column not in data.columns:
|
| 795 |
+
return None, score_missing, None, score_missing
|
| 796 |
+
|
| 797 |
+
price_fig, price_message = _pareto_axis(
|
| 798 |
+
data,
|
| 799 |
+
score_column,
|
| 800 |
+
_PARETO_PRICE_COLUMN,
|
| 801 |
+
"Price per image (USD)",
|
| 802 |
+
"Price per image isn't available for this dataset.",
|
| 803 |
+
"No models have both a score and a price for this metric.",
|
| 804 |
+
x_hover_prefix="$",
|
| 805 |
)
|
| 806 |
+
time_fig, time_message = _pareto_axis(
|
| 807 |
+
data,
|
| 808 |
+
score_column,
|
| 809 |
+
_PARETO_TIME_COLUMN,
|
| 810 |
+
"Min generation time (s)",
|
| 811 |
+
"Min generation time isn't available for this dataset.",
|
| 812 |
+
"No models have both a score and a min generation time for this metric.",
|
| 813 |
+
x_hover_suffix="s",
|
| 814 |
+
)
|
| 815 |
+
return price_fig, price_message, time_fig, time_message
|
| 816 |
+
|
| 817 |
+
|
| 818 |
+
def _pareto_dataset_message(data):
|
| 819 |
+
has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
|
| 820 |
+
has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
|
| 821 |
+
if has_price or has_time:
|
| 822 |
+
return None
|
| 823 |
+
return (
|
| 824 |
+
"Price per image and min generation time aren't available for "
|
| 825 |
+
"this dataset, so these plots can't be drawn."
|
| 826 |
)
|
| 827 |
|
| 828 |
|
| 829 |
+
def _pareto_slot_note(price_fig, price_message, time_fig, time_message, data):
|
| 830 |
+
has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
|
| 831 |
+
has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
|
| 832 |
+
notes = []
|
| 833 |
+
if has_price and not has_time:
|
| 834 |
+
notes.append(
|
| 835 |
+
"Min generation time isn't available for this dataset, so only "
|
| 836 |
+
"price vs score is shown."
|
| 837 |
+
)
|
| 838 |
+
elif has_time and not has_price:
|
| 839 |
+
notes.append(
|
| 840 |
+
"Price per image isn't available for this dataset, so only min "
|
| 841 |
+
"generation time vs score is shown."
|
| 842 |
+
)
|
| 843 |
+
if price_fig is None and has_price:
|
| 844 |
+
notes.append(price_message)
|
| 845 |
+
if time_fig is None and has_time:
|
| 846 |
+
notes.append(time_message)
|
| 847 |
+
if len(notes) == 2 and notes[0] == notes[1]:
|
| 848 |
+
notes = notes[:1]
|
| 849 |
+
return " ".join(notes)
|
| 850 |
+
|
| 851 |
+
|
| 852 |
+
def _pareto_slot_updates(data, score_columns):
|
| 853 |
+
"""Updates for a fixed bank of Gradio Plot slots (visible/hidden)."""
|
| 854 |
+
score_columns = [column for column in (score_columns or []) if column]
|
| 855 |
+
has_price = data is not None and _PARETO_PRICE_COLUMN in data.columns
|
| 856 |
+
has_time = data is not None and _PARETO_TIME_COLUMN in data.columns
|
| 857 |
+
dataset_note = _pareto_dataset_message(data)
|
| 858 |
+
updates = [_pareto_note_update(dataset_note)]
|
| 859 |
+
hide_all_slots = not has_price and not has_time
|
| 860 |
+
hidden_slot = (
|
| 861 |
+
gr.update(visible=False),
|
| 862 |
+
"",
|
| 863 |
+
_pareto_note_update(""),
|
| 864 |
+
gr.update(visible=False),
|
| 865 |
+
gr.update(visible=False),
|
| 866 |
+
_pareto_plot_update(None),
|
| 867 |
+
gr.update(visible=False),
|
| 868 |
+
_pareto_plot_update(None),
|
| 869 |
+
)
|
| 870 |
+
for index in range(MAX_PARETO_METRICS):
|
| 871 |
+
if hide_all_slots or index >= len(score_columns):
|
| 872 |
+
updates.extend(hidden_slot)
|
| 873 |
+
continue
|
| 874 |
+
score_column = score_columns[index]
|
| 875 |
+
price_fig, price_message, time_fig, time_message = _pareto_pair(
|
| 876 |
+
data, score_column
|
| 877 |
+
)
|
| 878 |
+
show_price = price_fig is not None
|
| 879 |
+
show_time = time_fig is not None
|
| 880 |
+
updates.extend(
|
| 881 |
+
[
|
| 882 |
+
gr.update(visible=True),
|
| 883 |
+
f"#### {_display_label(score_column)}",
|
| 884 |
+
_pareto_note_update(
|
| 885 |
+
_pareto_slot_note(
|
| 886 |
+
price_fig,
|
| 887 |
+
price_message,
|
| 888 |
+
time_fig,
|
| 889 |
+
time_message,
|
| 890 |
+
data,
|
| 891 |
+
)
|
| 892 |
+
),
|
| 893 |
+
gr.update(visible=show_price or show_time),
|
| 894 |
+
gr.update(visible=show_price),
|
| 895 |
+
_pareto_plot_update(price_fig),
|
| 896 |
+
gr.update(visible=show_time),
|
| 897 |
+
_pareto_plot_update(time_fig),
|
| 898 |
+
]
|
| 899 |
+
)
|
| 900 |
+
return updates
|
| 901 |
+
|
| 902 |
+
|
| 903 |
+
def _samples_html(samples, selected_models, num_prompts, seed=0):
|
| 904 |
+
if not samples:
|
| 905 |
+
return _pareto_unavailable_html(
|
| 906 |
+
"Samples aren't available for this dataset."
|
| 907 |
+
)
|
| 908 |
+
images = samples.get("images", {})
|
| 909 |
+
models = [model for model in (selected_models or []) if model in images]
|
| 910 |
+
if not models:
|
| 911 |
+
models = (samples.get("models") or [])[:2]
|
| 912 |
+
return _build_compare_samples_html(samples, models, num_prompts, seed)
|
| 913 |
+
|
| 914 |
+
|
| 915 |
def _build_compare_samples_html(samples, selected_models, num_prompts, seed=0):
|
| 916 |
+
selected_models = list(selected_models or [])[:MAX_COMPARE_MODELS]
|
|
|
|
|
|
|
|
|
|
|
|
|
| 917 |
|
| 918 |
if not selected_models:
|
| 919 |
return (
|
|
|
|
| 954 |
cells.append(
|
| 955 |
f"""
|
| 956 |
<div class="compare-cell">
|
| 957 |
+
<div class="compare-model-label">{escape(display_model_name(model))}</div>
|
| 958 |
<a href="{image_url}" target="_blank" rel="noopener noreferrer">
|
| 959 |
+
<img src="{image_url}" alt="{escape(display_model_name(model))} sample" loading="lazy" />
|
| 960 |
</a>
|
| 961 |
</div>
|
| 962 |
"""
|
|
|
|
| 979 |
return "\n".join(blocks)
|
| 980 |
|
| 981 |
|
| 982 |
+
def _leaderboard_intro_markdown(note):
|
| 983 |
+
parts = [
|
| 984 |
+
"Models are ranked by the selected metric, with price and generation "
|
| 985 |
+
"time in the same table."
|
| 986 |
+
]
|
| 987 |
+
extra = (note or "").strip()
|
| 988 |
+
if extra:
|
| 989 |
+
parts.append(extra)
|
| 990 |
+
return "<p class='view-help'>" + " ".join(parts) + "</p>"
|
| 991 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 992 |
|
| 993 |
+
def _filter_row(datasets, metrics, default_dataset_id, default_metric_id=None):
|
| 994 |
+
metric_id = _coerce_metric(
|
| 995 |
+
datasets, metrics, default_dataset_id, default_metric_id
|
| 996 |
+
)
|
| 997 |
+
with gr.Row(elem_classes="view-filters"):
|
| 998 |
+
dataset_dd = gr.Dropdown(
|
| 999 |
+
choices=_dataset_choices(datasets),
|
| 1000 |
+
value=default_dataset_id,
|
| 1001 |
+
label="Dataset",
|
| 1002 |
+
type="value",
|
| 1003 |
+
filterable=False,
|
| 1004 |
+
scale=2,
|
| 1005 |
+
min_width=160,
|
| 1006 |
)
|
| 1007 |
+
metric_dd = gr.Dropdown(
|
| 1008 |
+
choices=_metric_dropdown_choices(datasets, metrics, default_dataset_id),
|
| 1009 |
+
value=_metric_dropdown_value(metric_id),
|
| 1010 |
+
label="Metric",
|
| 1011 |
+
type="value",
|
| 1012 |
+
multiselect=True,
|
| 1013 |
+
allow_custom_value=False,
|
| 1014 |
+
filterable=True,
|
| 1015 |
+
scale=2,
|
| 1016 |
+
min_width=180,
|
| 1017 |
+
elem_classes="filter-chips",
|
| 1018 |
+
)
|
| 1019 |
+
models_dd = gr.Dropdown(
|
| 1020 |
+
choices=_model_choices(datasets, default_dataset_id),
|
| 1021 |
+
value=[],
|
| 1022 |
+
multiselect=True,
|
| 1023 |
+
label="Models",
|
| 1024 |
+
type="value",
|
| 1025 |
+
allow_custom_value=False,
|
| 1026 |
+
filterable=True,
|
| 1027 |
+
scale=2,
|
| 1028 |
+
min_width=180,
|
| 1029 |
+
elem_classes="filter-chips",
|
| 1030 |
+
)
|
| 1031 |
+
return dataset_dd, metric_dd, models_dd
|
| 1032 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1033 |
|
| 1034 |
+
def render_image_workspace(datasets, metrics, default_dataset_id, default_metric_id):
|
| 1035 |
+
default_metric_id = _coerce_metric(
|
| 1036 |
+
datasets, metrics, default_dataset_id, default_metric_id
|
| 1037 |
+
)
|
| 1038 |
+
initial = resolve_view(datasets, metrics, default_dataset_id, default_metric_id)
|
| 1039 |
+
initial_data = initial["data"]
|
| 1040 |
+
initial_columns = initial["columns"]
|
| 1041 |
+
initial_samples = initial.get("samples")
|
| 1042 |
+
with gr.Column(elem_classes="workspace-shell"):
|
| 1043 |
+
with gr.Column(elem_classes="workspace-filters") as filters_host:
|
| 1044 |
+
gr.Markdown(
|
| 1045 |
+
"<p class='filter-help'>"
|
| 1046 |
+
"These filters apply to Leaderboards, Pareto plots, and Samples. "
|
| 1047 |
+
"On Samples, only datasets we have generations for are listed. "
|
| 1048 |
+
"On Pareto plots, only datasets with price or generation time "
|
| 1049 |
+
"are listed. Search in Models, or leave it empty to include "
|
| 1050 |
+
"every model."
|
| 1051 |
+
"</p>",
|
| 1052 |
+
elem_classes="filter-help-host",
|
| 1053 |
+
)
|
| 1054 |
+
dataset_dd, metric_dd, models_dd = _filter_row(
|
| 1055 |
+
datasets, metrics, default_dataset_id, None
|
| 1056 |
+
)
|
| 1057 |
+
with gr.Tabs(elem_classes="main-tabs") as main_tabs:
|
| 1058 |
+
with gr.TabItem("Leaderboards", id=TAB_LEADERBOARDS) as lb_tab:
|
| 1059 |
+
lb_note = gr.Markdown(
|
| 1060 |
+
_leaderboard_intro_markdown(initial.get("note")),
|
| 1061 |
+
elem_classes="view-help-host",
|
| 1062 |
+
)
|
| 1063 |
+
platform_choices = _filter_choices(initial_data, "Platform")
|
| 1064 |
+
owner_choices = _filter_choices(initial_data, "Endpoint Owner")
|
| 1065 |
+
optimized_choices = _filter_choices(initial_data, "Optimized")
|
| 1066 |
+
with gr.Row(
|
| 1067 |
+
elem_classes="leaderboard-controls",
|
| 1068 |
+
visible=bool(
|
| 1069 |
+
platform_choices or owner_choices or optimized_choices
|
| 1070 |
+
),
|
| 1071 |
+
) as lb_controls:
|
| 1072 |
+
platform = gr.Dropdown(
|
| 1073 |
+
choices=platform_choices,
|
| 1074 |
+
value=[],
|
| 1075 |
+
label="Providers",
|
| 1076 |
+
multiselect=True,
|
| 1077 |
+
allow_custom_value=False,
|
| 1078 |
+
filterable=False,
|
| 1079 |
+
scale=1,
|
| 1080 |
+
visible=bool(platform_choices),
|
| 1081 |
+
)
|
| 1082 |
+
owner = gr.Dropdown(
|
| 1083 |
+
choices=owner_choices,
|
| 1084 |
+
value=[],
|
| 1085 |
+
label="Endpoint owners",
|
| 1086 |
+
multiselect=True,
|
| 1087 |
+
allow_custom_value=False,
|
| 1088 |
+
filterable=False,
|
| 1089 |
+
scale=1,
|
| 1090 |
+
visible=bool(owner_choices),
|
| 1091 |
+
)
|
| 1092 |
+
optimized = gr.Dropdown(
|
| 1093 |
+
choices=optimized_choices,
|
| 1094 |
+
value=[],
|
| 1095 |
+
label="Optimized",
|
| 1096 |
+
multiselect=True,
|
| 1097 |
+
allow_custom_value=False,
|
| 1098 |
+
filterable=False,
|
| 1099 |
+
scale=1,
|
| 1100 |
+
visible=bool(optimized_choices),
|
| 1101 |
+
)
|
| 1102 |
+
ranking = gr.HTML(
|
| 1103 |
+
_leaderboard_html(
|
| 1104 |
+
_assign_leaderboard_ranks(
|
| 1105 |
+
initial_data, initial.get("score_column")
|
| 1106 |
+
),
|
| 1107 |
+
initial_columns,
|
| 1108 |
+
),
|
| 1109 |
+
padding=False,
|
| 1110 |
+
elem_classes="ranking-table-host",
|
| 1111 |
+
)
|
| 1112 |
|
| 1113 |
+
with gr.TabItem(
|
| 1114 |
+
"Pareto Plots",
|
| 1115 |
+
id=TAB_PARETO,
|
| 1116 |
+
interactive=_dataset_has_pareto(datasets, default_dataset_id),
|
| 1117 |
+
) as pp_tab:
|
| 1118 |
+
gr.Markdown(
|
| 1119 |
+
"<p class='view-help'>"
|
| 1120 |
+
"Score against price and generation time. Green points are on the "
|
| 1121 |
+
"frontier; lavender points sit below it. Hover a point to see "
|
| 1122 |
+
"which model it is."
|
| 1123 |
+
"</p>",
|
| 1124 |
+
elem_classes="view-help-host",
|
| 1125 |
+
)
|
| 1126 |
+
pareto_dataset_note = gr.HTML(
|
| 1127 |
+
"",
|
| 1128 |
+
padding=False,
|
| 1129 |
+
visible=False,
|
| 1130 |
+
elem_classes="pareto-note",
|
| 1131 |
+
)
|
| 1132 |
+
pareto_slots = []
|
| 1133 |
+
for slot_index in range(MAX_PARETO_METRICS):
|
| 1134 |
+
with gr.Column(
|
| 1135 |
+
visible=False,
|
| 1136 |
+
elem_classes="pareto-metric-block",
|
| 1137 |
+
) as slot_group:
|
| 1138 |
+
slot_title = gr.Markdown(
|
| 1139 |
+
"",
|
| 1140 |
+
elem_classes="pareto-metric-title",
|
| 1141 |
+
)
|
| 1142 |
+
with gr.Row(
|
| 1143 |
+
equal_height=True,
|
| 1144 |
+
elem_classes="pareto-layout",
|
| 1145 |
+
) as slot_layout:
|
| 1146 |
+
with gr.Column(
|
| 1147 |
+
scale=1,
|
| 1148 |
+
min_width=320,
|
| 1149 |
+
elem_classes="pareto-col",
|
| 1150 |
+
) as slot_price_col:
|
| 1151 |
+
gr.Markdown(
|
| 1152 |
+
"#### Price vs score",
|
| 1153 |
+
elem_classes="pareto-subhead",
|
| 1154 |
+
)
|
| 1155 |
+
slot_price = gr.Plot(
|
| 1156 |
+
value=None,
|
| 1157 |
+
show_label=False,
|
| 1158 |
+
elem_classes="pareto-plot",
|
| 1159 |
+
)
|
| 1160 |
+
with gr.Column(
|
| 1161 |
+
scale=1,
|
| 1162 |
+
min_width=320,
|
| 1163 |
+
elem_classes="pareto-col",
|
| 1164 |
+
) as slot_time_col:
|
| 1165 |
+
gr.Markdown(
|
| 1166 |
+
"#### Min generation time vs score",
|
| 1167 |
+
elem_classes="pareto-subhead",
|
| 1168 |
+
)
|
| 1169 |
+
slot_time = gr.Plot(
|
| 1170 |
+
value=None,
|
| 1171 |
+
show_label=False,
|
| 1172 |
+
elem_classes="pareto-plot",
|
| 1173 |
+
)
|
| 1174 |
+
slot_note = gr.HTML(
|
| 1175 |
+
"",
|
| 1176 |
+
padding=False,
|
| 1177 |
+
visible=False,
|
| 1178 |
+
elem_classes="pareto-note",
|
| 1179 |
+
)
|
| 1180 |
+
pareto_slots.append(
|
| 1181 |
+
(
|
| 1182 |
+
slot_group,
|
| 1183 |
+
slot_title,
|
| 1184 |
+
slot_note,
|
| 1185 |
+
slot_layout,
|
| 1186 |
+
slot_price_col,
|
| 1187 |
+
slot_price,
|
| 1188 |
+
slot_time_col,
|
| 1189 |
+
slot_time,
|
| 1190 |
+
)
|
| 1191 |
+
)
|
| 1192 |
|
| 1193 |
+
with gr.TabItem(
|
| 1194 |
+
"Samples",
|
| 1195 |
+
id=TAB_SAMPLES,
|
| 1196 |
+
interactive=_dataset_has_samples(datasets, default_dataset_id),
|
| 1197 |
+
) as sm_tab:
|
| 1198 |
+
with gr.Column(visible=bool(initial_samples)) as samples_panel:
|
| 1199 |
+
gr.Markdown(
|
| 1200 |
+
f"<p class='view-help'>"
|
| 1201 |
+
f"The same prompts, side by side. Select up to "
|
| 1202 |
+
f"<strong>{MAX_COMPARE_MODELS}</strong> models above, or leave "
|
| 1203 |
+
f"Models empty for two defaults."
|
| 1204 |
+
f"</p>",
|
| 1205 |
+
elem_classes="view-help-host",
|
| 1206 |
+
)
|
| 1207 |
+
with gr.Row(equal_height=False, elem_classes="compare-controls"):
|
| 1208 |
+
prompt_count = gr.Slider(
|
| 1209 |
+
minimum=1,
|
| 1210 |
+
maximum=MAX_COMPARE_PROMPTS,
|
| 1211 |
+
value=DEFAULT_COMPARE_PROMPTS,
|
| 1212 |
+
step=1,
|
| 1213 |
+
label="Prompts to show",
|
| 1214 |
+
container=False,
|
| 1215 |
+
show_reset_button=False,
|
| 1216 |
+
scale=1,
|
| 1217 |
+
min_width=180,
|
| 1218 |
+
elem_classes="compare-prompt-count",
|
| 1219 |
+
)
|
| 1220 |
+
shuffle_button = gr.Button(
|
| 1221 |
+
"Shuffle prompts",
|
| 1222 |
+
variant="primary",
|
| 1223 |
+
scale=0,
|
| 1224 |
+
min_width=140,
|
| 1225 |
+
elem_classes="compare-shuffle",
|
| 1226 |
+
)
|
| 1227 |
+
gallery = gr.HTML(
|
| 1228 |
+
value=_samples_html(
|
| 1229 |
+
initial_samples, [], DEFAULT_COMPARE_PROMPTS, seed=0
|
| 1230 |
+
),
|
| 1231 |
+
elem_classes="compare-gallery",
|
| 1232 |
+
)
|
| 1233 |
+
seed_state = gr.State(0)
|
| 1234 |
+
|
| 1235 |
+
with gr.TabItem("About", id=TAB_ABOUT) as about_tab:
|
| 1236 |
+
render_about()
|
| 1237 |
+
|
| 1238 |
+
def _synced_filters(dataset_id, metric_id, models, *, clear_metric=False):
|
| 1239 |
+
if clear_metric:
|
| 1240 |
+
metric_id = []
|
| 1241 |
+
else:
|
| 1242 |
+
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 1243 |
+
model_choices = _model_choices(datasets, dataset_id)
|
| 1244 |
+
model_values = set(_model_choice_values(model_choices))
|
| 1245 |
+
models = [model for model in (models or []) if model in model_values]
|
| 1246 |
+
metric_choices = _metric_dropdown_choices(datasets, metrics, dataset_id)
|
| 1247 |
return (
|
| 1248 |
+
dataset_id,
|
| 1249 |
+
metric_id,
|
| 1250 |
+
models,
|
| 1251 |
+
gr.update(
|
| 1252 |
+
choices=metric_choices,
|
| 1253 |
+
value=_metric_dropdown_value(metric_id),
|
| 1254 |
+
),
|
| 1255 |
+
gr.update(choices=model_choices, value=models),
|
| 1256 |
)
|
| 1257 |
|
| 1258 |
+
def _leaderboard_extras(data, platform_value, owner_value, optimized_value):
|
| 1259 |
+
platform_choices = _filter_choices(data, "Platform")
|
| 1260 |
+
owner_choices = _filter_choices(data, "Endpoint Owner")
|
| 1261 |
+
optimized_choices = _filter_choices(data, "Optimized")
|
| 1262 |
+
platform_value = [
|
| 1263 |
+
value for value in (platform_value or []) if value in platform_choices
|
| 1264 |
+
]
|
| 1265 |
+
owner_value = [
|
| 1266 |
+
value for value in (owner_value or []) if value in owner_choices
|
| 1267 |
+
]
|
| 1268 |
+
optimized_value = [
|
| 1269 |
+
value for value in (optimized_value or []) if value in optimized_choices
|
| 1270 |
+
]
|
| 1271 |
+
return (
|
| 1272 |
+
gr.update(
|
| 1273 |
+
choices=platform_choices,
|
| 1274 |
+
value=platform_value,
|
| 1275 |
+
visible=bool(platform_choices),
|
| 1276 |
+
),
|
| 1277 |
+
gr.update(
|
| 1278 |
+
choices=owner_choices,
|
| 1279 |
+
value=owner_value,
|
| 1280 |
+
visible=bool(owner_choices),
|
| 1281 |
+
),
|
| 1282 |
+
gr.update(
|
| 1283 |
+
choices=optimized_choices,
|
| 1284 |
+
value=optimized_value,
|
| 1285 |
+
visible=bool(optimized_choices),
|
| 1286 |
+
),
|
| 1287 |
+
platform_value,
|
| 1288 |
+
owner_value,
|
| 1289 |
+
optimized_value,
|
| 1290 |
+
gr.update(
|
| 1291 |
+
visible=bool(
|
| 1292 |
+
platform_choices or owner_choices or optimized_choices
|
| 1293 |
+
)
|
| 1294 |
+
),
|
| 1295 |
)
|
| 1296 |
|
| 1297 |
+
def _content_flags(tab):
|
| 1298 |
+
return {
|
| 1299 |
+
"include_leaderboard": tab == TAB_LEADERBOARDS,
|
| 1300 |
+
"include_pareto": tab == TAB_PARETO,
|
| 1301 |
+
"include_samples": tab == TAB_SAMPLES,
|
| 1302 |
+
}
|
| 1303 |
+
|
| 1304 |
+
def _commit_state(
|
| 1305 |
+
view_state,
|
| 1306 |
+
dataset_id,
|
| 1307 |
+
metric_id,
|
| 1308 |
+
models,
|
| 1309 |
+
tab,
|
| 1310 |
+
flags,
|
| 1311 |
+
extras=None,
|
| 1312 |
+
):
|
| 1313 |
+
prev = dict(view_state or {})
|
| 1314 |
+
extras = extras or {}
|
| 1315 |
+
return {
|
| 1316 |
+
"dataset_id": dataset_id,
|
| 1317 |
+
"metric_id": metric_id,
|
| 1318 |
+
"models": list(models or []),
|
| 1319 |
+
"current_tab": tab,
|
| 1320 |
+
"platform": list(
|
| 1321 |
+
extras.get("platform", prev.get("platform") or [])
|
| 1322 |
+
),
|
| 1323 |
+
"owner": list(extras.get("owner", prev.get("owner") or [])),
|
| 1324 |
+
"optimized": list(
|
| 1325 |
+
extras.get("optimized", prev.get("optimized") or [])
|
| 1326 |
+
),
|
| 1327 |
+
"stale": {
|
| 1328 |
+
TAB_LEADERBOARDS: not flags["include_leaderboard"],
|
| 1329 |
+
TAB_PARETO: not flags["include_pareto"],
|
| 1330 |
+
TAB_SAMPLES: not flags["include_samples"],
|
| 1331 |
+
},
|
| 1332 |
+
}
|
| 1333 |
+
|
| 1334 |
+
def _save_leaderboard_filters(
|
| 1335 |
+
view_state, platform_value, owner_value, optimized_value
|
| 1336 |
+
):
|
| 1337 |
+
view_state["platform"] = list(platform_value or [])
|
| 1338 |
+
view_state["owner"] = list(owner_value or [])
|
| 1339 |
+
view_state["optimized"] = list(optimized_value or [])
|
| 1340 |
+
return view_state
|
| 1341 |
|
| 1342 |
+
def _restore_leaderboard_filters(
|
| 1343 |
+
view_state, platform_value, owner_value, optimized_value
|
| 1344 |
+
):
|
| 1345 |
+
stored_platform = (view_state or {}).get("platform") or []
|
| 1346 |
+
stored_owner = (view_state or {}).get("owner") or []
|
| 1347 |
+
stored_optimized = (view_state or {}).get("optimized") or []
|
| 1348 |
+
view = resolve_view(
|
| 1349 |
+
datasets,
|
| 1350 |
+
metrics,
|
| 1351 |
+
(view_state or {}).get("dataset_id"),
|
| 1352 |
+
(view_state or {}).get("metric_id"),
|
| 1353 |
+
)
|
| 1354 |
+
extras = _leaderboard_extras(
|
| 1355 |
+
view["data"] if view else None,
|
| 1356 |
+
stored_platform,
|
| 1357 |
+
stored_owner,
|
| 1358 |
+
stored_optimized,
|
| 1359 |
+
)
|
| 1360 |
+
platform_update = extras[0] if list(platform_value or []) != extras[3] else gr.skip()
|
| 1361 |
+
owner_update = extras[1] if list(owner_value or []) != extras[4] else gr.skip()
|
| 1362 |
+
optimized_update = extras[2] if list(optimized_value or []) != extras[5] else gr.skip()
|
| 1363 |
+
return (
|
| 1364 |
+
extras[6],
|
| 1365 |
+
platform_update,
|
| 1366 |
+
owner_update,
|
| 1367 |
+
optimized_update,
|
| 1368 |
+
extras[3],
|
| 1369 |
+
extras[4],
|
| 1370 |
+
extras[5],
|
| 1371 |
+
)
|
| 1372 |
|
| 1373 |
+
def _views(
|
| 1374 |
+
dataset_id,
|
| 1375 |
+
metric_id,
|
| 1376 |
+
models,
|
| 1377 |
+
platform_value,
|
| 1378 |
+
owner_value,
|
| 1379 |
+
optimized_value,
|
| 1380 |
+
num_prompts,
|
| 1381 |
+
seed,
|
| 1382 |
+
*,
|
| 1383 |
+
include_leaderboard=True,
|
| 1384 |
+
include_pareto=False,
|
| 1385 |
+
include_samples=False,
|
| 1386 |
+
):
|
| 1387 |
+
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1388 |
+
data = view["data"]
|
| 1389 |
+
if include_leaderboard:
|
| 1390 |
+
note = _leaderboard_intro_markdown(view.get("note"))
|
| 1391 |
+
ranking_html = _leaderboard_html(
|
| 1392 |
+
_filter_leaderboard(
|
| 1393 |
+
_assign_leaderboard_ranks(data, view["score_column"]),
|
| 1394 |
+
platform_value or [],
|
| 1395 |
+
owner_value or [],
|
| 1396 |
+
optimized_value or [],
|
| 1397 |
+
models=models,
|
| 1398 |
+
),
|
| 1399 |
+
view["columns"],
|
| 1400 |
+
)
|
| 1401 |
+
else:
|
| 1402 |
+
note = gr.skip()
|
| 1403 |
+
ranking_html = gr.skip()
|
| 1404 |
+
if include_pareto:
|
| 1405 |
+
pareto_data = _filter_leaderboard(data, [], [], [], models=models)
|
| 1406 |
+
pareto_updates = _pareto_slot_updates(pareto_data, view["score_columns"])
|
| 1407 |
+
else:
|
| 1408 |
+
pareto_updates = _pareto_skip_updates()
|
| 1409 |
+
if include_samples:
|
| 1410 |
+
samples = view.get("samples")
|
| 1411 |
+
samples_html = _samples_html(
|
| 1412 |
+
samples,
|
| 1413 |
+
models,
|
| 1414 |
+
int(num_prompts or DEFAULT_COMPARE_PROMPTS),
|
| 1415 |
+
int(seed or 0),
|
| 1416 |
+
)
|
| 1417 |
+
samples_visible = gr.update(visible=bool(samples))
|
| 1418 |
+
else:
|
| 1419 |
+
samples_html = gr.skip()
|
| 1420 |
+
samples_visible = gr.skip()
|
| 1421 |
+
return (
|
| 1422 |
+
note,
|
| 1423 |
+
ranking_html,
|
| 1424 |
+
*pareto_updates,
|
| 1425 |
+
samples_html,
|
| 1426 |
+
samples_visible,
|
| 1427 |
+
)
|
| 1428 |
|
| 1429 |
+
def _apply_filter_change(
|
| 1430 |
+
source,
|
| 1431 |
+
dataset_id,
|
| 1432 |
+
metric_id,
|
| 1433 |
+
models,
|
| 1434 |
+
platform_value,
|
| 1435 |
+
owner_value,
|
| 1436 |
+
optimized_value,
|
| 1437 |
+
num_prompts,
|
| 1438 |
+
seed,
|
| 1439 |
+
view_state,
|
| 1440 |
+
):
|
| 1441 |
+
view_state = dict(view_state or {})
|
| 1442 |
+
tab = view_state.get("current_tab") or TAB_LEADERBOARDS
|
| 1443 |
+
selected_raw = _normalize_metric_ids(metric_id)
|
| 1444 |
+
incoming_models = list(models or [])
|
| 1445 |
+
dataset_changed = source == "dataset" and dataset_id != view_state.get(
|
| 1446 |
+
"dataset_id"
|
| 1447 |
+
)
|
| 1448 |
|
| 1449 |
+
if source == "dataset":
|
| 1450 |
+
synced = _synced_filters(
|
| 1451 |
+
dataset_id, metric_id, models, clear_metric=dataset_changed
|
| 1452 |
+
)
|
| 1453 |
+
dataset_id, metric_id, models = synced[:3]
|
| 1454 |
+
metric_update, models_update = synced[3], synced[4]
|
| 1455 |
+
else:
|
| 1456 |
+
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 1457 |
+
if source == "models":
|
| 1458 |
+
allowed = set(
|
| 1459 |
+
_model_choice_values(_model_choices(datasets, dataset_id))
|
| 1460 |
+
)
|
| 1461 |
+
models = [model for model in incoming_models if model in allowed]
|
| 1462 |
+
models_update = (
|
| 1463 |
+
gr.update(value=models)
|
| 1464 |
+
if models != incoming_models
|
| 1465 |
+
else gr.skip()
|
| 1466 |
+
)
|
| 1467 |
+
else:
|
| 1468 |
+
models = incoming_models
|
| 1469 |
+
models_update = gr.skip()
|
| 1470 |
+
metric_update = (
|
| 1471 |
+
gr.update(
|
| 1472 |
+
choices=_metric_dropdown_choices(datasets, metrics, dataset_id),
|
| 1473 |
+
value=_metric_dropdown_value(metric_id),
|
| 1474 |
+
)
|
| 1475 |
+
if source == "metric" and ALL_METRICS_ID in selected_raw
|
| 1476 |
+
else gr.skip()
|
| 1477 |
+
)
|
| 1478 |
|
| 1479 |
+
unchanged = _applied_key(view_state) == _selection_key(
|
| 1480 |
+
dataset_id, metric_id, models
|
| 1481 |
+
)
|
| 1482 |
+
if (
|
| 1483 |
+
not dataset_changed
|
| 1484 |
+
and not (source == "metric" and ALL_METRICS_ID in selected_raw)
|
| 1485 |
+
and unchanged
|
| 1486 |
+
):
|
| 1487 |
+
return None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1488 |
|
| 1489 |
+
selected_tab = tab
|
| 1490 |
+
extras = (
|
| 1491 |
+
list(platform_value or []),
|
| 1492 |
+
list(owner_value or []),
|
| 1493 |
+
list(optimized_value or []),
|
| 1494 |
+
)
|
| 1495 |
+
extra_updates = None
|
| 1496 |
+
can_pareto = _dataset_has_pareto(datasets, dataset_id)
|
| 1497 |
+
can_samples = _dataset_has_samples(datasets, dataset_id)
|
| 1498 |
+
if source == "dataset":
|
| 1499 |
+
if tab == TAB_SAMPLES and not can_samples:
|
| 1500 |
+
selected_tab = TAB_LEADERBOARDS
|
| 1501 |
+
elif tab == TAB_PARETO and not can_pareto:
|
| 1502 |
+
selected_tab = TAB_LEADERBOARDS
|
| 1503 |
+
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1504 |
+
extra_updates = _leaderboard_extras(
|
| 1505 |
+
view["data"] if view else None,
|
| 1506 |
+
platform_value,
|
| 1507 |
+
owner_value,
|
| 1508 |
+
optimized_value,
|
| 1509 |
+
)
|
| 1510 |
+
extras = extra_updates[3:6]
|
| 1511 |
|
| 1512 |
+
flags = _content_flags(selected_tab)
|
| 1513 |
+
extras_payload = (
|
| 1514 |
+
{
|
| 1515 |
+
"platform": extras[0],
|
| 1516 |
+
"owner": extras[1],
|
| 1517 |
+
"optimized": extras[2],
|
| 1518 |
+
}
|
| 1519 |
+
if selected_tab == TAB_LEADERBOARDS
|
| 1520 |
+
else {}
|
| 1521 |
+
)
|
| 1522 |
+
return {
|
| 1523 |
+
"dataset_id": dataset_id,
|
| 1524 |
+
"metric_update": metric_update,
|
| 1525 |
+
"models_update": models_update,
|
| 1526 |
+
"selected_tab": selected_tab,
|
| 1527 |
+
"tab": tab,
|
| 1528 |
+
"extra_updates": extra_updates,
|
| 1529 |
+
"can_pareto": can_pareto,
|
| 1530 |
+
"can_samples": can_samples,
|
| 1531 |
+
"views": _views(
|
| 1532 |
+
dataset_id,
|
| 1533 |
+
metric_id,
|
| 1534 |
+
models,
|
| 1535 |
+
extras[0],
|
| 1536 |
+
extras[1],
|
| 1537 |
+
extras[2],
|
| 1538 |
+
num_prompts,
|
| 1539 |
+
seed,
|
| 1540 |
+
**flags,
|
| 1541 |
+
),
|
| 1542 |
+
"state": _commit_state(
|
| 1543 |
+
view_state,
|
| 1544 |
+
dataset_id,
|
| 1545 |
+
metric_id,
|
| 1546 |
+
models,
|
| 1547 |
+
selected_tab,
|
| 1548 |
+
flags,
|
| 1549 |
+
extras=extras_payload,
|
| 1550 |
+
),
|
| 1551 |
+
}
|
| 1552 |
+
|
| 1553 |
+
def on_dataset(
|
| 1554 |
+
dataset_id,
|
| 1555 |
+
metric_id,
|
| 1556 |
+
models,
|
| 1557 |
+
platform_value,
|
| 1558 |
+
owner_value,
|
| 1559 |
+
optimized_value,
|
| 1560 |
+
num_prompts,
|
| 1561 |
+
seed,
|
| 1562 |
+
view_state,
|
| 1563 |
+
):
|
| 1564 |
+
result = _apply_filter_change(
|
| 1565 |
+
"dataset",
|
| 1566 |
+
dataset_id,
|
| 1567 |
+
metric_id,
|
| 1568 |
+
models,
|
| 1569 |
+
platform_value,
|
| 1570 |
+
owner_value,
|
| 1571 |
+
optimized_value,
|
| 1572 |
+
num_prompts,
|
| 1573 |
+
seed,
|
| 1574 |
+
view_state,
|
| 1575 |
+
)
|
| 1576 |
+
if result is None:
|
| 1577 |
+
return _skip_all(len(dataset_outputs))
|
| 1578 |
+
extras = result["extra_updates"]
|
| 1579 |
+
return (
|
| 1580 |
+
_dataset_dropdown_update(
|
| 1581 |
+
datasets, result["selected_tab"], result["dataset_id"]
|
| 1582 |
+
),
|
| 1583 |
+
result["metric_update"],
|
| 1584 |
+
result["models_update"],
|
| 1585 |
+
extras[6],
|
| 1586 |
+
extras[0],
|
| 1587 |
+
extras[1],
|
| 1588 |
+
extras[2],
|
| 1589 |
+
*result["views"],
|
| 1590 |
+
gr.update(interactive=result["can_pareto"]),
|
| 1591 |
+
gr.update(interactive=result["can_samples"]),
|
| 1592 |
+
gr.update(selected=result["selected_tab"])
|
| 1593 |
+
if result["selected_tab"] != result["tab"]
|
| 1594 |
+
else gr.skip(),
|
| 1595 |
+
result["state"],
|
| 1596 |
+
)
|
| 1597 |
|
| 1598 |
+
def on_metric(
|
| 1599 |
+
dataset_id,
|
| 1600 |
+
metric_id,
|
| 1601 |
+
models,
|
| 1602 |
+
platform_value,
|
| 1603 |
+
owner_value,
|
| 1604 |
+
optimized_value,
|
| 1605 |
+
num_prompts,
|
| 1606 |
+
seed,
|
| 1607 |
+
view_state,
|
| 1608 |
+
):
|
| 1609 |
+
result = _apply_filter_change(
|
| 1610 |
+
"metric",
|
| 1611 |
+
dataset_id,
|
| 1612 |
+
metric_id,
|
| 1613 |
+
models,
|
| 1614 |
+
platform_value,
|
| 1615 |
+
owner_value,
|
| 1616 |
+
optimized_value,
|
| 1617 |
+
num_prompts,
|
| 1618 |
+
seed,
|
| 1619 |
+
view_state,
|
| 1620 |
+
)
|
| 1621 |
+
if result is None:
|
| 1622 |
+
return _skip_all(len(metric_outputs))
|
| 1623 |
+
return (result["metric_update"], *result["views"], result["state"])
|
| 1624 |
+
|
| 1625 |
+
def on_models(
|
| 1626 |
+
dataset_id,
|
| 1627 |
+
metric_id,
|
| 1628 |
+
models,
|
| 1629 |
+
platform_value,
|
| 1630 |
+
owner_value,
|
| 1631 |
+
optimized_value,
|
| 1632 |
+
num_prompts,
|
| 1633 |
+
seed,
|
| 1634 |
+
view_state,
|
| 1635 |
+
):
|
| 1636 |
+
result = _apply_filter_change(
|
| 1637 |
+
"models",
|
| 1638 |
+
dataset_id,
|
| 1639 |
+
metric_id,
|
| 1640 |
+
models,
|
| 1641 |
+
platform_value,
|
| 1642 |
+
owner_value,
|
| 1643 |
+
optimized_value,
|
| 1644 |
+
num_prompts,
|
| 1645 |
+
seed,
|
| 1646 |
+
view_state,
|
| 1647 |
+
)
|
| 1648 |
+
if result is None:
|
| 1649 |
+
return _skip_all(len(models_outputs))
|
| 1650 |
+
return (result["models_update"], *result["views"], result["state"])
|
| 1651 |
+
|
| 1652 |
+
def on_tab_select(
|
| 1653 |
+
tab,
|
| 1654 |
+
dataset_id,
|
| 1655 |
+
metric_id,
|
| 1656 |
+
models,
|
| 1657 |
+
platform_value,
|
| 1658 |
+
owner_value,
|
| 1659 |
+
optimized_value,
|
| 1660 |
+
num_prompts,
|
| 1661 |
+
seed,
|
| 1662 |
+
view_state,
|
| 1663 |
+
):
|
| 1664 |
+
view_state = dict(view_state or {})
|
| 1665 |
+
prev_tab = view_state.get("current_tab") or TAB_LEADERBOARDS
|
| 1666 |
+
requested_tab = tab
|
| 1667 |
+
if tab == TAB_SAMPLES and not _dataset_has_samples(datasets, dataset_id):
|
| 1668 |
+
tab = TAB_LEADERBOARDS
|
| 1669 |
+
elif tab == TAB_PARETO and not _dataset_has_pareto(datasets, dataset_id):
|
| 1670 |
+
tab = TAB_LEADERBOARDS
|
| 1671 |
+
if prev_tab == TAB_LEADERBOARDS:
|
| 1672 |
+
_save_leaderboard_filters(
|
| 1673 |
+
view_state,
|
| 1674 |
+
platform_value,
|
| 1675 |
+
owner_value,
|
| 1676 |
+
optimized_value,
|
| 1677 |
)
|
| 1678 |
+
dataset_update = _dataset_dropdown_update(datasets, tab, dataset_id)
|
| 1679 |
+
metric_id = _coerce_metric(datasets, metrics, dataset_id, metric_id)
|
| 1680 |
+
models = list(models or [])
|
| 1681 |
+
view_state["current_tab"] = tab
|
| 1682 |
+
view_state["dataset_id"] = dataset_id
|
| 1683 |
+
view_state["metric_id"] = metric_id
|
| 1684 |
+
view_state["models"] = models
|
| 1685 |
+
show_filters = tab != TAB_ABOUT
|
| 1686 |
+
show_metric = tab in (TAB_LEADERBOARDS, TAB_PARETO)
|
| 1687 |
+
was_filters = prev_tab != TAB_ABOUT
|
| 1688 |
+
was_metric = prev_tab in (TAB_LEADERBOARDS, TAB_PARETO)
|
| 1689 |
+
filters_vis = (
|
| 1690 |
+
gr.update(visible=show_filters)
|
| 1691 |
+
if show_filters != was_filters
|
| 1692 |
+
else gr.skip()
|
| 1693 |
)
|
| 1694 |
+
metric_vis = (
|
| 1695 |
+
gr.update(visible=show_metric)
|
| 1696 |
+
if show_metric != was_metric
|
| 1697 |
+
else gr.skip()
|
| 1698 |
+
)
|
| 1699 |
+
if tab == TAB_LEADERBOARDS:
|
| 1700 |
+
restored = _restore_leaderboard_filters(
|
| 1701 |
+
view_state,
|
| 1702 |
+
platform_value,
|
| 1703 |
+
owner_value,
|
| 1704 |
+
optimized_value,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1705 |
)
|
| 1706 |
+
platform_value = restored[4]
|
| 1707 |
+
owner_value = restored[5]
|
| 1708 |
+
optimized_value = restored[6]
|
| 1709 |
+
lb_filters = restored[:4]
|
| 1710 |
+
else:
|
| 1711 |
+
lb_filters = _skip_all(4)
|
| 1712 |
+
stale = dict(view_state.get("stale") or {})
|
| 1713 |
+
chrome = (
|
| 1714 |
+
filters_vis,
|
| 1715 |
+
dataset_update,
|
| 1716 |
+
metric_vis,
|
| 1717 |
+
gr.skip(),
|
| 1718 |
+
*lb_filters,
|
| 1719 |
+
)
|
| 1720 |
+
tab_select = (
|
| 1721 |
+
gr.update(selected=tab)
|
| 1722 |
+
if tab != requested_tab
|
| 1723 |
+
else gr.skip()
|
| 1724 |
+
)
|
| 1725 |
+
if tab == TAB_ABOUT or not stale.get(tab, True):
|
| 1726 |
+
return (
|
| 1727 |
+
*chrome,
|
| 1728 |
+
*_skip_all(len(view_outputs)),
|
| 1729 |
+
tab_select,
|
| 1730 |
+
view_state,
|
| 1731 |
+
)
|
| 1732 |
+
flags = _content_flags(tab)
|
| 1733 |
+
views = _views(
|
| 1734 |
+
dataset_id,
|
| 1735 |
+
metric_id,
|
| 1736 |
+
models,
|
| 1737 |
+
platform_value,
|
| 1738 |
+
owner_value,
|
| 1739 |
+
optimized_value,
|
| 1740 |
+
num_prompts,
|
| 1741 |
+
seed,
|
| 1742 |
+
**flags,
|
| 1743 |
+
)
|
| 1744 |
+
stale[tab] = False
|
| 1745 |
+
view_state["stale"] = stale
|
| 1746 |
+
return (*chrome, *views, tab_select, view_state)
|
| 1747 |
+
|
| 1748 |
+
def on_leaderboard_filters(
|
| 1749 |
+
dataset_id,
|
| 1750 |
+
metric_id,
|
| 1751 |
+
models,
|
| 1752 |
+
platform_value,
|
| 1753 |
+
owner_value,
|
| 1754 |
+
optimized_value,
|
| 1755 |
+
view_state,
|
| 1756 |
+
):
|
| 1757 |
+
view_state = dict(view_state or {})
|
| 1758 |
+
_save_leaderboard_filters(
|
| 1759 |
+
view_state,
|
| 1760 |
+
platform_value,
|
| 1761 |
+
owner_value,
|
| 1762 |
+
optimized_value,
|
| 1763 |
+
)
|
| 1764 |
+
view = resolve_view(datasets, metrics, dataset_id, metric_id)
|
| 1765 |
+
return (
|
| 1766 |
+
_leaderboard_html(
|
| 1767 |
+
_filter_leaderboard(
|
| 1768 |
+
_assign_leaderboard_ranks(view["data"], view["score_column"]),
|
| 1769 |
+
platform_value or [],
|
| 1770 |
+
owner_value or [],
|
| 1771 |
+
optimized_value or [],
|
| 1772 |
+
models=models,
|
| 1773 |
+
),
|
| 1774 |
+
view["columns"],
|
| 1775 |
+
),
|
| 1776 |
+
view_state,
|
| 1777 |
)
|
| 1778 |
|
| 1779 |
+
def on_samples_controls(dataset_id, models, num_prompts, seed):
|
| 1780 |
+
view = resolve_view(datasets, metrics, dataset_id, None)
|
| 1781 |
+
return _samples_html(
|
| 1782 |
+
view.get("samples") if view else None,
|
| 1783 |
+
models,
|
| 1784 |
+
int(num_prompts or DEFAULT_COMPARE_PROMPTS),
|
| 1785 |
+
int(seed or 0),
|
| 1786 |
+
)
|
| 1787 |
+
|
| 1788 |
+
def on_shuffle(dataset_id, models, num_prompts, seed):
|
| 1789 |
+
next_seed = int(seed or 0) + 1
|
| 1790 |
+
view = resolve_view(datasets, metrics, dataset_id, None)
|
| 1791 |
+
return next_seed, _samples_html(
|
| 1792 |
+
view.get("samples") if view else None,
|
| 1793 |
+
models,
|
| 1794 |
+
int(num_prompts or DEFAULT_COMPARE_PROMPTS),
|
| 1795 |
+
next_seed,
|
| 1796 |
+
)
|
| 1797 |
+
|
| 1798 |
+
def _on_tab(tab):
|
| 1799 |
+
def handler(
|
| 1800 |
+
dataset_id,
|
| 1801 |
+
metric_id,
|
| 1802 |
+
models,
|
| 1803 |
+
platform_value,
|
| 1804 |
+
owner_value,
|
| 1805 |
+
optimized_value,
|
| 1806 |
+
num_prompts,
|
| 1807 |
+
seed,
|
| 1808 |
+
view_state,
|
| 1809 |
+
):
|
| 1810 |
+
return on_tab_select(
|
| 1811 |
+
tab,
|
| 1812 |
+
dataset_id,
|
| 1813 |
+
metric_id,
|
| 1814 |
+
models,
|
| 1815 |
+
platform_value,
|
| 1816 |
+
owner_value,
|
| 1817 |
+
optimized_value,
|
| 1818 |
+
num_prompts,
|
| 1819 |
+
seed,
|
| 1820 |
+
view_state,
|
| 1821 |
+
)
|
| 1822 |
+
|
| 1823 |
+
handler.__name__ = f"on_tab_{tab}"
|
| 1824 |
+
return handler
|
| 1825 |
+
|
| 1826 |
+
view_state = gr.State(
|
| 1827 |
+
{
|
| 1828 |
+
"dataset_id": default_dataset_id,
|
| 1829 |
+
"metric_id": None,
|
| 1830 |
+
"models": [],
|
| 1831 |
+
"current_tab": TAB_LEADERBOARDS,
|
| 1832 |
+
"platform": [],
|
| 1833 |
+
"owner": [],
|
| 1834 |
+
"optimized": [],
|
| 1835 |
+
"stale": {
|
| 1836 |
+
TAB_LEADERBOARDS: False,
|
| 1837 |
+
TAB_PARETO: True,
|
| 1838 |
+
TAB_SAMPLES: False,
|
| 1839 |
+
},
|
| 1840 |
+
}
|
| 1841 |
)
|
| 1842 |
+
pareto_outputs = [
|
| 1843 |
+
pareto_dataset_note,
|
| 1844 |
+
*[
|
| 1845 |
+
component
|
| 1846 |
+
for slot_group, slot_title, slot_note, slot_layout, slot_price_col, slot_price, slot_time_col, slot_time in pareto_slots
|
| 1847 |
+
for component in (
|
| 1848 |
+
slot_group,
|
| 1849 |
+
slot_title,
|
| 1850 |
+
slot_note,
|
| 1851 |
+
slot_layout,
|
| 1852 |
+
slot_price_col,
|
| 1853 |
+
slot_price,
|
| 1854 |
+
slot_time_col,
|
| 1855 |
+
slot_time,
|
| 1856 |
+
)
|
| 1857 |
+
],
|
| 1858 |
+
]
|
| 1859 |
+
view_inputs = [
|
| 1860 |
+
platform,
|
| 1861 |
+
owner,
|
| 1862 |
+
optimized,
|
| 1863 |
+
prompt_count,
|
| 1864 |
+
seed_state,
|
| 1865 |
+
view_state,
|
| 1866 |
+
]
|
| 1867 |
+
view_outputs = [
|
| 1868 |
+
lb_note,
|
| 1869 |
+
ranking,
|
| 1870 |
+
*pareto_outputs,
|
| 1871 |
+
gallery,
|
| 1872 |
+
samples_panel,
|
| 1873 |
+
]
|
| 1874 |
+
filter_inputs = [dataset_dd, metric_dd, models_dd, *view_inputs]
|
| 1875 |
+
|
| 1876 |
+
dataset_outputs = [
|
| 1877 |
+
dataset_dd,
|
| 1878 |
+
metric_dd,
|
| 1879 |
+
models_dd,
|
| 1880 |
+
lb_controls,
|
| 1881 |
+
platform,
|
| 1882 |
+
owner,
|
| 1883 |
+
optimized,
|
| 1884 |
+
*view_outputs,
|
| 1885 |
+
pp_tab,
|
| 1886 |
+
sm_tab,
|
| 1887 |
+
main_tabs,
|
| 1888 |
+
view_state,
|
| 1889 |
+
]
|
| 1890 |
+
dataset_dd.change(
|
| 1891 |
+
on_dataset,
|
| 1892 |
+
inputs=filter_inputs,
|
| 1893 |
+
outputs=dataset_outputs,
|
| 1894 |
+
**_VIEW_EVENTS,
|
| 1895 |
)
|
| 1896 |
+
|
| 1897 |
+
metric_outputs = [
|
| 1898 |
+
metric_dd,
|
| 1899 |
+
*view_outputs,
|
| 1900 |
+
view_state,
|
| 1901 |
+
]
|
| 1902 |
+
metric_dd.change(
|
| 1903 |
+
on_metric,
|
| 1904 |
+
inputs=filter_inputs,
|
| 1905 |
+
outputs=metric_outputs,
|
| 1906 |
+
**_VIEW_EVENTS,
|
| 1907 |
)
|
|
|
|
| 1908 |
|
| 1909 |
+
models_outputs = [
|
| 1910 |
+
models_dd,
|
| 1911 |
+
*view_outputs,
|
| 1912 |
+
view_state,
|
| 1913 |
+
]
|
| 1914 |
+
models_dd.change(
|
| 1915 |
+
on_models,
|
| 1916 |
+
inputs=filter_inputs,
|
| 1917 |
+
outputs=models_outputs,
|
| 1918 |
+
**_VIEW_EVENTS,
|
| 1919 |
+
)
|
| 1920 |
|
| 1921 |
+
tab_outputs = [
|
| 1922 |
+
filters_host,
|
| 1923 |
+
dataset_dd,
|
| 1924 |
+
metric_dd,
|
| 1925 |
+
models_dd,
|
| 1926 |
+
lb_controls,
|
| 1927 |
+
platform,
|
| 1928 |
+
owner,
|
| 1929 |
+
optimized,
|
| 1930 |
+
*view_outputs,
|
| 1931 |
+
main_tabs,
|
| 1932 |
+
view_state,
|
| 1933 |
]
|
| 1934 |
+
for tab, tab_item in (
|
| 1935 |
+
(TAB_LEADERBOARDS, lb_tab),
|
| 1936 |
+
(TAB_PARETO, pp_tab),
|
| 1937 |
+
(TAB_SAMPLES, sm_tab),
|
| 1938 |
+
(TAB_ABOUT, about_tab),
|
| 1939 |
+
):
|
| 1940 |
+
tab_item.select(
|
| 1941 |
+
_on_tab(tab),
|
| 1942 |
+
inputs=filter_inputs,
|
| 1943 |
+
outputs=tab_outputs,
|
| 1944 |
+
show_progress="hidden",
|
| 1945 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1946 |
|
| 1947 |
+
for component in (platform, owner, optimized):
|
| 1948 |
+
component.change(
|
| 1949 |
+
on_leaderboard_filters,
|
| 1950 |
+
inputs=[
|
| 1951 |
+
dataset_dd,
|
| 1952 |
+
metric_dd,
|
| 1953 |
+
models_dd,
|
| 1954 |
+
platform,
|
| 1955 |
+
owner,
|
| 1956 |
+
optimized,
|
| 1957 |
+
view_state,
|
| 1958 |
+
],
|
| 1959 |
+
outputs=[ranking, view_state],
|
| 1960 |
+
show_progress="hidden",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1961 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1962 |
|
| 1963 |
+
prompt_count.change(
|
| 1964 |
+
on_samples_controls,
|
| 1965 |
+
inputs=[dataset_dd, models_dd, prompt_count, seed_state],
|
| 1966 |
+
outputs=gallery,
|
| 1967 |
+
show_progress="hidden",
|
| 1968 |
+
)
|
| 1969 |
+
shuffle_button.click(
|
| 1970 |
+
on_shuffle,
|
| 1971 |
+
inputs=[dataset_dd, models_dd, prompt_count, seed_state],
|
| 1972 |
+
outputs=[seed_state, gallery],
|
| 1973 |
+
show_progress="hidden",
|
| 1974 |
+
)
|
| 1975 |
|
| 1976 |
def render_about():
|
| 1977 |
+
with gr.Row(elem_classes="about-layout", equal_height=False):
|
| 1978 |
+
with gr.Column(scale=1, min_width=340, elem_classes="about-col"):
|
| 1979 |
+
gr.Markdown(ABOUT_OVERVIEW_CONTENT, elem_classes="about-copy")
|
| 1980 |
+
with gr.Column(scale=1, min_width=340, elem_classes="about-col"):
|
| 1981 |
+
gr.Markdown(ABOUT_DETAILS_CONTENT, elem_classes="about-copy")
|
| 1982 |
|
| 1983 |
|
| 1984 |
def render_footer():
|
| 1985 |
+
gr.HTML(FOOTER_CONTENT, padding=False, elem_classes="page-footer-host")
|
| 1986 |
+
with gr.Accordion(
|
| 1987 |
+
"Citation",
|
| 1988 |
+
open=False,
|
| 1989 |
+
elem_classes="citation-accordion",
|
| 1990 |
+
):
|
| 1991 |
gr.Markdown(CITATION_CONTENT)
|