Spaces:
Running
Running
Commit ·
51cb5c8
1
Parent(s): cdb54bf
fix traces
Browse files- assets/webpagebench.css +17 -0
- src/traces.py +143 -8
assets/webpagebench.css
CHANGED
|
@@ -1857,6 +1857,23 @@ footer {
|
|
| 1857 |
color: var(--text-muted);
|
| 1858 |
}
|
| 1859 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1860 |
.gradio-container .wab-trace-detail-view .wab-trace-detail-tab,
|
| 1861 |
.gradio-container .wab-trace-detail-view .wab-trace-step-item,
|
| 1862 |
.gradio-container .wab-trace-detail-view .wab-trace-export,
|
|
|
|
| 1857 |
color: var(--text-muted);
|
| 1858 |
}
|
| 1859 |
|
| 1860 |
+
.wab-trace-step-description {
|
| 1861 |
+
margin-bottom: 0.65rem;
|
| 1862 |
+
font-size: 0.86rem;
|
| 1863 |
+
line-height: 1.45;
|
| 1864 |
+
color: var(--text);
|
| 1865 |
+
white-space: pre-wrap;
|
| 1866 |
+
}
|
| 1867 |
+
|
| 1868 |
+
.wab-trace-step-shot {
|
| 1869 |
+
display: block;
|
| 1870 |
+
max-width: 100%;
|
| 1871 |
+
height: auto;
|
| 1872 |
+
margin: 0.35rem 0 0.75rem;
|
| 1873 |
+
border: 1px solid var(--border);
|
| 1874 |
+
border-radius: var(--radius);
|
| 1875 |
+
}
|
| 1876 |
+
|
| 1877 |
.gradio-container .wab-trace-detail-view .wab-trace-detail-tab,
|
| 1878 |
.gradio-container .wab-trace-detail-view .wab-trace-step-item,
|
| 1879 |
.gradio-container .wab-trace-detail-view .wab-trace-export,
|
src/traces.py
CHANGED
|
@@ -2,9 +2,12 @@
|
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
|
|
|
| 5 |
import html
|
|
|
|
| 6 |
import json
|
| 7 |
import re
|
|
|
|
| 8 |
from typing import Any
|
| 9 |
|
| 10 |
from src.bench_config import load_bench_config
|
|
@@ -121,26 +124,42 @@ def _format_browser_action(model_output: dict[str, Any] | None) -> str:
|
|
| 121 |
return ""
|
| 122 |
|
| 123 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
def _normalize_agent_step(step: dict[str, Any], harness: str) -> dict[str, str]:
|
| 125 |
harness = (harness or "").lower()
|
| 126 |
title = t("step_prefix")
|
| 127 |
body_parts: list[str] = []
|
| 128 |
action = ""
|
| 129 |
url = ""
|
|
|
|
|
|
|
| 130 |
|
| 131 |
if "model_output" in step:
|
| 132 |
model_output = step.get("model_output") or {}
|
|
|
|
|
|
|
| 133 |
title = f"{t('step_prefix')} {step.get('step', step.get('metadata', {}).get('step_number', '?'))}"
|
| 134 |
action = _format_browser_action(model_output)
|
| 135 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
if thinking:
|
| 137 |
-
body_parts.append(
|
| 138 |
results = step.get("result")
|
| 139 |
if isinstance(results, list):
|
| 140 |
for row in results[:2]:
|
| 141 |
if isinstance(row, dict) and row.get("extracted_content"):
|
| 142 |
body_parts.append(str(row["extracted_content"]))
|
| 143 |
-
url = _extract_url_from_state(step.get("state")) or ""
|
|
|
|
| 144 |
if not body_parts and step.get("state_message"):
|
| 145 |
body_parts.append(_truncate(str(step["state_message"]), 400))
|
| 146 |
elif step.get("kind") == "message":
|
|
@@ -160,11 +179,102 @@ def _normalize_agent_step(step: dict[str, Any], harness: str) -> dict[str, str]:
|
|
| 160 |
return {
|
| 161 |
"title": title,
|
| 162 |
"action": action,
|
|
|
|
| 163 |
"body": _truncate("\n\n".join(part for part in body_parts if part), 1200),
|
| 164 |
"url": url,
|
|
|
|
| 165 |
}
|
| 166 |
|
| 167 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
def _step_sidebar_label(step: dict[str, Any], harness: str) -> str:
|
| 169 |
if "model_output" in step:
|
| 170 |
formatted = _format_browser_action(step.get("model_output") or {})
|
|
@@ -198,8 +308,10 @@ def _ui_agent_steps(test: dict[str, Any], harness: str) -> list[dict[str, str]]:
|
|
| 198 |
"kind": "user",
|
| 199 |
"title": t("task_title"),
|
| 200 |
"action": "",
|
|
|
|
| 201 |
"body": str(test.get("input") or ""),
|
| 202 |
"url": str(test.get("task_url") or ""),
|
|
|
|
| 203 |
}
|
| 204 |
]
|
| 205 |
for raw_step in agent_steps:
|
|
@@ -215,18 +327,29 @@ def _ui_agent_steps(test: dict[str, Any], harness: str) -> list[dict[str, str]]:
|
|
| 215 |
"kind": _step_kind_class(raw_step),
|
| 216 |
"title": norm["title"],
|
| 217 |
"action": norm["action"],
|
|
|
|
| 218 |
"body": norm["body"],
|
| 219 |
"url": norm["url"],
|
|
|
|
| 220 |
}
|
| 221 |
)
|
| 222 |
return rows
|
| 223 |
|
| 224 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 225 |
def _extract_action_rows(test: dict[str, Any], harness: str) -> list[dict[str, str]]:
|
| 226 |
rows: list[dict[str, str]] = []
|
| 227 |
for step in _ui_agent_steps(test, harness)[1:]:
|
| 228 |
-
|
| 229 |
-
|
|
|
|
| 230 |
extended = test.get("extended_metrics") or {}
|
| 231 |
if not rows and extended.get("action_count"):
|
| 232 |
rows.append(
|
|
@@ -585,9 +708,14 @@ def build_trace_detail(
|
|
| 585 |
step_panels: list[str] = []
|
| 586 |
for step in ui_steps:
|
| 587 |
active = " active" if step["id"] == "0" else ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 588 |
step_nav_items.append(
|
| 589 |
f'<button type="button" class="wab-trace-step-item{active}" data-wab-trace-step="{html.escape(step["id"])}" '
|
| 590 |
-
f'data-wab-step-label="{html.escape(
|
| 591 |
f'<span class="wab-trace-step-num">{int(step["id"]) + 1}</span>'
|
| 592 |
f'<span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>'
|
| 593 |
f'<span class="wab-trace-step-label">{_esc(step["label"])}</span>'
|
|
@@ -599,12 +727,19 @@ def build_trace_detail(
|
|
| 599 |
if step["action"]
|
| 600 |
else ""
|
| 601 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 602 |
step_panels.append(
|
| 603 |
f'<div class="wab-trace-step-panel{active}" data-wab-trace-step-panel="{html.escape(step["id"])}">'
|
| 604 |
f'<div class="wab-trace-step-panel-head"><span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>'
|
| 605 |
f"<strong>{_esc(step['title'])}</strong></div>"
|
| 606 |
-
f"{url_line}{action_line}"
|
| 607 |
-
f'<pre class="wab-trace-step-body">{_esc(step["body"])}</pre>'
|
| 608 |
"</div>"
|
| 609 |
)
|
| 610 |
|
|
|
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
| 5 |
+
import base64
|
| 6 |
import html
|
| 7 |
+
import io
|
| 8 |
import json
|
| 9 |
import re
|
| 10 |
+
from pathlib import Path
|
| 11 |
from typing import Any
|
| 12 |
|
| 13 |
from src.bench_config import load_bench_config
|
|
|
|
| 124 |
return ""
|
| 125 |
|
| 126 |
|
| 127 |
+
def _step_text(value: object) -> str:
|
| 128 |
+
if isinstance(value, str):
|
| 129 |
+
return value.strip()
|
| 130 |
+
return ""
|
| 131 |
+
|
| 132 |
+
|
| 133 |
def _normalize_agent_step(step: dict[str, Any], harness: str) -> dict[str, str]:
|
| 134 |
harness = (harness or "").lower()
|
| 135 |
title = t("step_prefix")
|
| 136 |
body_parts: list[str] = []
|
| 137 |
action = ""
|
| 138 |
url = ""
|
| 139 |
+
description = ""
|
| 140 |
+
screenshot = ""
|
| 141 |
|
| 142 |
if "model_output" in step:
|
| 143 |
model_output = step.get("model_output") or {}
|
| 144 |
+
if not isinstance(model_output, dict):
|
| 145 |
+
model_output = {}
|
| 146 |
title = f"{t('step_prefix')} {step.get('step', step.get('metadata', {}).get('step_number', '?'))}"
|
| 147 |
action = _format_browser_action(model_output)
|
| 148 |
+
description = _step_text(model_output.get("action_description"))
|
| 149 |
+
thinking = (
|
| 150 |
+
_step_text(model_output.get("thinking"))
|
| 151 |
+
or _step_text(model_output.get("thought"))
|
| 152 |
+
or _step_text(model_output.get("next_goal"))
|
| 153 |
+
)
|
| 154 |
if thinking:
|
| 155 |
+
body_parts.append(thinking)
|
| 156 |
results = step.get("result")
|
| 157 |
if isinstance(results, list):
|
| 158 |
for row in results[:2]:
|
| 159 |
if isinstance(row, dict) and row.get("extracted_content"):
|
| 160 |
body_parts.append(str(row["extracted_content"]))
|
| 161 |
+
url = _extract_url_from_state(step.get("state")) or _step_text(step.get("url"))
|
| 162 |
+
screenshot = _step_text(step.get("screenshot")) or _step_text(step.get("screenshot_path"))
|
| 163 |
if not body_parts and step.get("state_message"):
|
| 164 |
body_parts.append(_truncate(str(step["state_message"]), 400))
|
| 165 |
elif step.get("kind") == "message":
|
|
|
|
| 179 |
return {
|
| 180 |
"title": title,
|
| 181 |
"action": action,
|
| 182 |
+
"action_description": description,
|
| 183 |
"body": _truncate("\n\n".join(part for part in body_parts if part), 1200),
|
| 184 |
"url": url,
|
| 185 |
+
"screenshot": screenshot,
|
| 186 |
}
|
| 187 |
|
| 188 |
|
| 189 |
+
def _allowed_screenshot_roots() -> list[Path]:
|
| 190 |
+
here = Path(__file__).resolve()
|
| 191 |
+
roots = [Path.cwd().resolve(), here.parents[1]]
|
| 192 |
+
repo = here.parents[2]
|
| 193 |
+
if (repo / "tests").is_dir() or (repo / "liderboard").is_dir():
|
| 194 |
+
roots.append(repo.resolve())
|
| 195 |
+
unique: list[Path] = []
|
| 196 |
+
for root in roots:
|
| 197 |
+
if root not in unique:
|
| 198 |
+
unique.append(root)
|
| 199 |
+
return unique
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
def _resolve_screenshot(path: str) -> Path | None:
|
| 203 |
+
raw = Path(path)
|
| 204 |
+
candidates = [raw] if raw.is_absolute() else [root / raw for root in _allowed_screenshot_roots()]
|
| 205 |
+
for candidate in candidates:
|
| 206 |
+
try:
|
| 207 |
+
resolved = candidate.resolve()
|
| 208 |
+
except OSError:
|
| 209 |
+
continue
|
| 210 |
+
if not resolved.is_file():
|
| 211 |
+
continue
|
| 212 |
+
if resolved.suffix.lower() not in {".png", ".jpg", ".jpeg", ".webp", ".gif"}:
|
| 213 |
+
continue
|
| 214 |
+
if not any(_is_relative_to(resolved, root) for root in _allowed_screenshot_roots()):
|
| 215 |
+
continue
|
| 216 |
+
return resolved
|
| 217 |
+
return None
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _is_relative_to(path: Path, root: Path) -> bool:
|
| 221 |
+
try:
|
| 222 |
+
path.relative_to(root)
|
| 223 |
+
except ValueError:
|
| 224 |
+
return False
|
| 225 |
+
return True
|
| 226 |
+
|
| 227 |
+
|
| 228 |
+
def _screenshot_html(path: str) -> str:
|
| 229 |
+
text = (path or "").strip()
|
| 230 |
+
if not text:
|
| 231 |
+
return ""
|
| 232 |
+
if text.startswith("data:image/"):
|
| 233 |
+
return (
|
| 234 |
+
'<img class="wab-trace-step-shot" alt="screenshot" loading="lazy" '
|
| 235 |
+
f'src="{html.escape(text, quote=True)}" />'
|
| 236 |
+
)
|
| 237 |
+
resolved = _resolve_screenshot(text)
|
| 238 |
+
if resolved is None:
|
| 239 |
+
return ""
|
| 240 |
+
src = _screenshot_data_uri(resolved)
|
| 241 |
+
if not src:
|
| 242 |
+
return ""
|
| 243 |
+
return (
|
| 244 |
+
'<img class="wab-trace-step-shot" alt="screenshot" loading="lazy" '
|
| 245 |
+
f'src="{src}" />'
|
| 246 |
+
)
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
def _screenshot_data_uri(path: Path) -> str:
|
| 250 |
+
try:
|
| 251 |
+
from PIL import Image
|
| 252 |
+
|
| 253 |
+
with Image.open(path) as img:
|
| 254 |
+
img = img.convert("RGB")
|
| 255 |
+
img.thumbnail((1100, 720))
|
| 256 |
+
buf = io.BytesIO()
|
| 257 |
+
img.save(buf, format="JPEG", quality=72)
|
| 258 |
+
encoded = base64.b64encode(buf.getvalue()).decode("ascii")
|
| 259 |
+
return f"data:image/jpeg;base64,{encoded}"
|
| 260 |
+
except Exception:
|
| 261 |
+
try:
|
| 262 |
+
data = path.read_bytes()
|
| 263 |
+
except OSError:
|
| 264 |
+
return ""
|
| 265 |
+
if not data or len(data) > 1_500_000:
|
| 266 |
+
return ""
|
| 267 |
+
mime = {
|
| 268 |
+
".png": "image/png",
|
| 269 |
+
".jpg": "image/jpeg",
|
| 270 |
+
".jpeg": "image/jpeg",
|
| 271 |
+
".webp": "image/webp",
|
| 272 |
+
".gif": "image/gif",
|
| 273 |
+
}.get(path.suffix.lower(), "image/png")
|
| 274 |
+
encoded = base64.b64encode(data).decode("ascii")
|
| 275 |
+
return f"data:{mime};base64,{encoded}"
|
| 276 |
+
|
| 277 |
+
|
| 278 |
def _step_sidebar_label(step: dict[str, Any], harness: str) -> str:
|
| 279 |
if "model_output" in step:
|
| 280 |
formatted = _format_browser_action(step.get("model_output") or {})
|
|
|
|
| 308 |
"kind": "user",
|
| 309 |
"title": t("task_title"),
|
| 310 |
"action": "",
|
| 311 |
+
"action_description": "",
|
| 312 |
"body": str(test.get("input") or ""),
|
| 313 |
"url": str(test.get("task_url") or ""),
|
| 314 |
+
"screenshot": "",
|
| 315 |
}
|
| 316 |
]
|
| 317 |
for raw_step in agent_steps:
|
|
|
|
| 327 |
"kind": _step_kind_class(raw_step),
|
| 328 |
"title": norm["title"],
|
| 329 |
"action": norm["action"],
|
| 330 |
+
"action_description": norm["action_description"],
|
| 331 |
"body": norm["body"],
|
| 332 |
"url": norm["url"],
|
| 333 |
+
"screenshot": norm["screenshot"],
|
| 334 |
}
|
| 335 |
)
|
| 336 |
return rows
|
| 337 |
|
| 338 |
|
| 339 |
+
def _action_row_detail(step: dict[str, str]) -> str:
|
| 340 |
+
action = step.get("action") or ""
|
| 341 |
+
description = step.get("action_description") or ""
|
| 342 |
+
if description and description != action:
|
| 343 |
+
return f"{action}\n{description}".strip()
|
| 344 |
+
return action
|
| 345 |
+
|
| 346 |
+
|
| 347 |
def _extract_action_rows(test: dict[str, Any], harness: str) -> list[dict[str, str]]:
|
| 348 |
rows: list[dict[str, str]] = []
|
| 349 |
for step in _ui_agent_steps(test, harness)[1:]:
|
| 350 |
+
detail = _action_row_detail(step)
|
| 351 |
+
if detail:
|
| 352 |
+
rows.append({"step": step["id"], "name": step["label"], "detail": detail})
|
| 353 |
extended = test.get("extended_metrics") or {}
|
| 354 |
if not rows and extended.get("action_count"):
|
| 355 |
rows.append(
|
|
|
|
| 708 |
step_panels: list[str] = []
|
| 709 |
for step in ui_steps:
|
| 710 |
active = " active" if step["id"] == "0" else ""
|
| 711 |
+
search_label = " ".join(
|
| 712 |
+
part
|
| 713 |
+
for part in (step["label"], step.get("action", ""), step.get("action_description", ""))
|
| 714 |
+
if part
|
| 715 |
+
)
|
| 716 |
step_nav_items.append(
|
| 717 |
f'<button type="button" class="wab-trace-step-item{active}" data-wab-trace-step="{html.escape(step["id"])}" '
|
| 718 |
+
f'data-wab-step-label="{html.escape(search_label.lower())}">'
|
| 719 |
f'<span class="wab-trace-step-num">{int(step["id"]) + 1}</span>'
|
| 720 |
f'<span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>'
|
| 721 |
f'<span class="wab-trace-step-label">{_esc(step["label"])}</span>'
|
|
|
|
| 727 |
if step["action"]
|
| 728 |
else ""
|
| 729 |
)
|
| 730 |
+
description = step.get("action_description") or ""
|
| 731 |
+
description_line = (
|
| 732 |
+
f'<div class="wab-trace-step-description">{_esc(description)}</div>'
|
| 733 |
+
if description and description != step.get("action")
|
| 734 |
+
else ""
|
| 735 |
+
)
|
| 736 |
+
body_html = f'<pre class="wab-trace-step-body">{_esc(step["body"])}</pre>' if step.get("body") else ""
|
| 737 |
+
shot_html = _screenshot_html(step.get("screenshot") or "")
|
| 738 |
step_panels.append(
|
| 739 |
f'<div class="wab-trace-step-panel{active}" data-wab-trace-step-panel="{html.escape(step["id"])}">'
|
| 740 |
f'<div class="wab-trace-step-panel-head"><span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>'
|
| 741 |
f"<strong>{_esc(step['title'])}</strong></div>"
|
| 742 |
+
f"{url_line}{action_line}{description_line}{shot_html}{body_html}"
|
|
|
|
| 743 |
"</div>"
|
| 744 |
)
|
| 745 |
|