Commit
·
8f49c6a
1
Parent(s):
8bd8d2f
Add quick select buttons for routing models (v0.3.15)
Browse files- Add QUICK_SELECT_MODELS list with popular models
- Add clickable buttons for quick model selection
- Support custom button labels (tuple format)
- Clean styling: white buttons with standard rounding
app.py
CHANGED
|
@@ -26,6 +26,20 @@ LITELLM_PRICES_CACHE = DATA_DIR / "litellm_prices.json"
|
|
| 26 |
S3_BUCKET = "s3://swe-bench-experiments/bash-only"
|
| 27 |
LITELLM_PRICES_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
|
| 28 |
LOG_DIR = Path("logs")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
| 30 |
LOG_FILE = LOG_DIR / "app.log"
|
| 31 |
|
|
@@ -1283,9 +1297,31 @@ def build_app():
|
|
| 1283 |
leaderboard_df = get_bash_only_df()
|
| 1284 |
|
| 1285 |
with gr.Blocks(title="SWE-bench Routing Cost Calculator") as app:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1286 |
trajectories_state = gr.State(None)
|
| 1287 |
|
| 1288 |
-
gr.Markdown("# 🧮 SWE-bench Bash-Only Leaderboard `v0.3.
|
| 1289 |
gr.Markdown("## 🎯 Select a base model for cost analysis (click a row)")
|
| 1290 |
|
| 1291 |
with gr.Row():
|
|
@@ -1373,6 +1409,16 @@ def build_app():
|
|
| 1373 |
with gr.Column():
|
| 1374 |
with gr.Group():
|
| 1375 |
gr.Markdown("#### Route to Model 1")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1376 |
routing_model_1 = gr.Dropdown(
|
| 1377 |
label="Model (type 3+ chars to search)",
|
| 1378 |
choices=[],
|
|
@@ -1390,6 +1436,16 @@ def build_app():
|
|
| 1390 |
with gr.Column(visible=False) as routing_block_2:
|
| 1391 |
with gr.Group():
|
| 1392 |
gr.Markdown("#### Route to Model 2")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1393 |
routing_model_2 = gr.Dropdown(
|
| 1394 |
label="Model (type 3+ chars to search)",
|
| 1395 |
choices=[],
|
|
@@ -1407,6 +1463,16 @@ def build_app():
|
|
| 1407 |
with gr.Column(visible=False) as routing_block_3:
|
| 1408 |
with gr.Group():
|
| 1409 |
gr.Markdown("#### Route to Model 3")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1410 |
routing_model_3 = gr.Dropdown(
|
| 1411 |
label="Model (type 3+ chars to search)",
|
| 1412 |
choices=[],
|
|
@@ -1524,6 +1590,20 @@ def build_app():
|
|
| 1524 |
routing_model_2.input(fn=filter_models, inputs=[routing_model_2], outputs=[routing_model_2])
|
| 1525 |
routing_model_3.input(fn=filter_models, inputs=[routing_model_3], outputs=[routing_model_3])
|
| 1526 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1527 |
def get_routing_prices_with_labels(model_name):
|
| 1528 |
"""Get all 4 prices for a routing model with found/estimated labels"""
|
| 1529 |
if not model_name:
|
|
|
|
| 26 |
S3_BUCKET = "s3://swe-bench-experiments/bash-only"
|
| 27 |
LITELLM_PRICES_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
|
| 28 |
LOG_DIR = Path("logs")
|
| 29 |
+
|
| 30 |
+
QUICK_SELECT_MODELS = [
|
| 31 |
+
"openrouter/anthropic/claude-opus-4.5",
|
| 32 |
+
"openrouter/anthropic/claude-sonnet-4.5",
|
| 33 |
+
"openrouter/google/gemini-3-pro-preview",
|
| 34 |
+
"openrouter/openai/gpt-5-codex",
|
| 35 |
+
"openrouter/openai/gpt-oss-120b",
|
| 36 |
+
"deepinfra/Qwen/Qwen3-14B",
|
| 37 |
+
"deepinfra/Qwen/Qwen3-32B",
|
| 38 |
+
"deepinfra/Qwen/Qwen3-73B",
|
| 39 |
+
"deepinfra/Qwen/Qwen3-235B-A22B",
|
| 40 |
+
"deepinfra/Qwen/Qwen3-30B-A3B",
|
| 41 |
+
("deepinfra/Qwen/Qwen3-Coder-480B-A35B-Instruct", "Qwen3-Coder-480B-A35B"),
|
| 42 |
+
]
|
| 43 |
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
| 44 |
LOG_FILE = LOG_DIR / "app.log"
|
| 45 |
|
|
|
|
| 1297 |
leaderboard_df = get_bash_only_df()
|
| 1298 |
|
| 1299 |
with gr.Blocks(title="SWE-bench Routing Cost Calculator") as app:
|
| 1300 |
+
gr.HTML("""
|
| 1301 |
+
<style>
|
| 1302 |
+
.quick-select-row {
|
| 1303 |
+
flex-wrap: wrap !important;
|
| 1304 |
+
gap: 6px !important;
|
| 1305 |
+
margin-bottom: 8px !important;
|
| 1306 |
+
}
|
| 1307 |
+
.quick-select-row button {
|
| 1308 |
+
background: white !important;
|
| 1309 |
+
color: #333 !important;
|
| 1310 |
+
border: 1px solid #ccc !important;
|
| 1311 |
+
border-radius: 4px !important;
|
| 1312 |
+
padding: 4px 10px !important;
|
| 1313 |
+
font-size: 12px !important;
|
| 1314 |
+
transition: all 0.15s ease !important;
|
| 1315 |
+
}
|
| 1316 |
+
.quick-select-row button:hover {
|
| 1317 |
+
background: #f0f0f0 !important;
|
| 1318 |
+
border-color: #999 !important;
|
| 1319 |
+
}
|
| 1320 |
+
</style>
|
| 1321 |
+
""")
|
| 1322 |
trajectories_state = gr.State(None)
|
| 1323 |
|
| 1324 |
+
gr.Markdown("# 🧮 SWE-bench Bash-Only Leaderboard `v0.3.15`")
|
| 1325 |
gr.Markdown("## 🎯 Select a base model for cost analysis (click a row)")
|
| 1326 |
|
| 1327 |
with gr.Row():
|
|
|
|
| 1409 |
with gr.Column():
|
| 1410 |
with gr.Group():
|
| 1411 |
gr.Markdown("#### Route to Model 1")
|
| 1412 |
+
with gr.Row(elem_classes=["quick-select-row"]):
|
| 1413 |
+
quick_btns_1 = []
|
| 1414 |
+
for item in QUICK_SELECT_MODELS:
|
| 1415 |
+
if isinstance(item, tuple):
|
| 1416 |
+
model, short_name = item
|
| 1417 |
+
else:
|
| 1418 |
+
model = item
|
| 1419 |
+
short_name = model.split("/")[-1]
|
| 1420 |
+
btn = gr.Button(short_name, size="sm", scale=0, min_width=80)
|
| 1421 |
+
quick_btns_1.append((btn, model))
|
| 1422 |
routing_model_1 = gr.Dropdown(
|
| 1423 |
label="Model (type 3+ chars to search)",
|
| 1424 |
choices=[],
|
|
|
|
| 1436 |
with gr.Column(visible=False) as routing_block_2:
|
| 1437 |
with gr.Group():
|
| 1438 |
gr.Markdown("#### Route to Model 2")
|
| 1439 |
+
with gr.Row(elem_classes=["quick-select-row"]):
|
| 1440 |
+
quick_btns_2 = []
|
| 1441 |
+
for item in QUICK_SELECT_MODELS:
|
| 1442 |
+
if isinstance(item, tuple):
|
| 1443 |
+
model, short_name = item
|
| 1444 |
+
else:
|
| 1445 |
+
model = item
|
| 1446 |
+
short_name = model.split("/")[-1]
|
| 1447 |
+
btn = gr.Button(short_name, size="sm", scale=0, min_width=80)
|
| 1448 |
+
quick_btns_2.append((btn, model))
|
| 1449 |
routing_model_2 = gr.Dropdown(
|
| 1450 |
label="Model (type 3+ chars to search)",
|
| 1451 |
choices=[],
|
|
|
|
| 1463 |
with gr.Column(visible=False) as routing_block_3:
|
| 1464 |
with gr.Group():
|
| 1465 |
gr.Markdown("#### Route to Model 3")
|
| 1466 |
+
with gr.Row(elem_classes=["quick-select-row"]):
|
| 1467 |
+
quick_btns_3 = []
|
| 1468 |
+
for item in QUICK_SELECT_MODELS:
|
| 1469 |
+
if isinstance(item, tuple):
|
| 1470 |
+
model, short_name = item
|
| 1471 |
+
else:
|
| 1472 |
+
model = item
|
| 1473 |
+
short_name = model.split("/")[-1]
|
| 1474 |
+
btn = gr.Button(short_name, size="sm", scale=0, min_width=80)
|
| 1475 |
+
quick_btns_3.append((btn, model))
|
| 1476 |
routing_model_3 = gr.Dropdown(
|
| 1477 |
label="Model (type 3+ chars to search)",
|
| 1478 |
choices=[],
|
|
|
|
| 1590 |
routing_model_2.input(fn=filter_models, inputs=[routing_model_2], outputs=[routing_model_2])
|
| 1591 |
routing_model_3.input(fn=filter_models, inputs=[routing_model_3], outputs=[routing_model_3])
|
| 1592 |
|
| 1593 |
+
def make_quick_select_fn(full_model_name):
|
| 1594 |
+
def fn():
|
| 1595 |
+
return gr.update(value=full_model_name)
|
| 1596 |
+
return fn
|
| 1597 |
+
|
| 1598 |
+
for btn, full_model in quick_btns_1:
|
| 1599 |
+
btn.click(fn=make_quick_select_fn(full_model), outputs=[routing_model_1])
|
| 1600 |
+
|
| 1601 |
+
for btn, full_model in quick_btns_2:
|
| 1602 |
+
btn.click(fn=make_quick_select_fn(full_model), outputs=[routing_model_2])
|
| 1603 |
+
|
| 1604 |
+
for btn, full_model in quick_btns_3:
|
| 1605 |
+
btn.click(fn=make_quick_select_fn(full_model), outputs=[routing_model_3])
|
| 1606 |
+
|
| 1607 |
def get_routing_prices_with_labels(model_name):
|
| 1608 |
"""Get all 4 prices for a routing model with found/estimated labels"""
|
| 1609 |
if not model_name:
|