ai-forever commited on
Commit
51cb5c8
·
1 Parent(s): cdb54bf

fix traces

Browse files
Files changed (2) hide show
  1. assets/webpagebench.css +17 -0
  2. src/traces.py +143 -8
assets/webpagebench.css CHANGED
@@ -1857,6 +1857,23 @@ footer {
1857
  color: var(--text-muted);
1858
  }
1859
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1860
  .gradio-container .wab-trace-detail-view .wab-trace-detail-tab,
1861
  .gradio-container .wab-trace-detail-view .wab-trace-step-item,
1862
  .gradio-container .wab-trace-detail-view .wab-trace-export,
 
1857
  color: var(--text-muted);
1858
  }
1859
 
1860
+ .wab-trace-step-description {
1861
+ margin-bottom: 0.65rem;
1862
+ font-size: 0.86rem;
1863
+ line-height: 1.45;
1864
+ color: var(--text);
1865
+ white-space: pre-wrap;
1866
+ }
1867
+
1868
+ .wab-trace-step-shot {
1869
+ display: block;
1870
+ max-width: 100%;
1871
+ height: auto;
1872
+ margin: 0.35rem 0 0.75rem;
1873
+ border: 1px solid var(--border);
1874
+ border-radius: var(--radius);
1875
+ }
1876
+
1877
  .gradio-container .wab-trace-detail-view .wab-trace-detail-tab,
1878
  .gradio-container .wab-trace-detail-view .wab-trace-step-item,
1879
  .gradio-container .wab-trace-detail-view .wab-trace-export,
src/traces.py CHANGED
@@ -2,9 +2,12 @@
2
 
3
  from __future__ import annotations
4
 
 
5
  import html
 
6
  import json
7
  import re
 
8
  from typing import Any
9
 
10
  from src.bench_config import load_bench_config
@@ -121,26 +124,42 @@ def _format_browser_action(model_output: dict[str, Any] | None) -> str:
121
  return ""
122
 
123
 
 
 
 
 
 
 
124
  def _normalize_agent_step(step: dict[str, Any], harness: str) -> dict[str, str]:
125
  harness = (harness or "").lower()
126
  title = t("step_prefix")
127
  body_parts: list[str] = []
128
  action = ""
129
  url = ""
 
 
130
 
131
  if "model_output" in step:
132
  model_output = step.get("model_output") or {}
 
 
133
  title = f"{t('step_prefix')} {step.get('step', step.get('metadata', {}).get('step_number', '?'))}"
134
  action = _format_browser_action(model_output)
135
- thinking = model_output.get("thinking") or model_output.get("next_goal")
 
 
 
 
 
136
  if thinking:
137
- body_parts.append(str(thinking))
138
  results = step.get("result")
139
  if isinstance(results, list):
140
  for row in results[:2]:
141
  if isinstance(row, dict) and row.get("extracted_content"):
142
  body_parts.append(str(row["extracted_content"]))
143
- url = _extract_url_from_state(step.get("state")) or ""
 
144
  if not body_parts and step.get("state_message"):
145
  body_parts.append(_truncate(str(step["state_message"]), 400))
146
  elif step.get("kind") == "message":
@@ -160,11 +179,102 @@ def _normalize_agent_step(step: dict[str, Any], harness: str) -> dict[str, str]:
160
  return {
161
  "title": title,
162
  "action": action,
 
163
  "body": _truncate("\n\n".join(part for part in body_parts if part), 1200),
164
  "url": url,
 
165
  }
166
 
167
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
168
  def _step_sidebar_label(step: dict[str, Any], harness: str) -> str:
169
  if "model_output" in step:
170
  formatted = _format_browser_action(step.get("model_output") or {})
@@ -198,8 +308,10 @@ def _ui_agent_steps(test: dict[str, Any], harness: str) -> list[dict[str, str]]:
198
  "kind": "user",
199
  "title": t("task_title"),
200
  "action": "",
 
201
  "body": str(test.get("input") or ""),
202
  "url": str(test.get("task_url") or ""),
 
203
  }
204
  ]
205
  for raw_step in agent_steps:
@@ -215,18 +327,29 @@ def _ui_agent_steps(test: dict[str, Any], harness: str) -> list[dict[str, str]]:
215
  "kind": _step_kind_class(raw_step),
216
  "title": norm["title"],
217
  "action": norm["action"],
 
218
  "body": norm["body"],
219
  "url": norm["url"],
 
220
  }
221
  )
222
  return rows
223
 
224
 
 
 
 
 
 
 
 
 
225
  def _extract_action_rows(test: dict[str, Any], harness: str) -> list[dict[str, str]]:
226
  rows: list[dict[str, str]] = []
227
  for step in _ui_agent_steps(test, harness)[1:]:
228
- if step["action"]:
229
- rows.append({"step": step["id"], "name": step["label"], "detail": step["action"]})
 
230
  extended = test.get("extended_metrics") or {}
231
  if not rows and extended.get("action_count"):
232
  rows.append(
@@ -585,9 +708,14 @@ def build_trace_detail(
585
  step_panels: list[str] = []
586
  for step in ui_steps:
587
  active = " active" if step["id"] == "0" else ""
 
 
 
 
 
588
  step_nav_items.append(
589
  f'<button type="button" class="wab-trace-step-item{active}" data-wab-trace-step="{html.escape(step["id"])}" '
590
- f'data-wab-step-label="{html.escape(step["label"].lower())}">'
591
  f'<span class="wab-trace-step-num">{int(step["id"]) + 1}</span>'
592
  f'<span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>'
593
  f'<span class="wab-trace-step-label">{_esc(step["label"])}</span>'
@@ -599,12 +727,19 @@ def build_trace_detail(
599
  if step["action"]
600
  else ""
601
  )
 
 
 
 
 
 
 
 
602
  step_panels.append(
603
  f'<div class="wab-trace-step-panel{active}" data-wab-trace-step-panel="{html.escape(step["id"])}">'
604
  f'<div class="wab-trace-step-panel-head"><span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>'
605
  f"<strong>{_esc(step['title'])}</strong></div>"
606
- f"{url_line}{action_line}"
607
- f'<pre class="wab-trace-step-body">{_esc(step["body"])}</pre>'
608
  "</div>"
609
  )
610
 
 
2
 
3
  from __future__ import annotations
4
 
5
+ import base64
6
  import html
7
+ import io
8
  import json
9
  import re
10
+ from pathlib import Path
11
  from typing import Any
12
 
13
  from src.bench_config import load_bench_config
 
124
  return ""
125
 
126
 
127
+ def _step_text(value: object) -> str:
128
+ if isinstance(value, str):
129
+ return value.strip()
130
+ return ""
131
+
132
+
133
  def _normalize_agent_step(step: dict[str, Any], harness: str) -> dict[str, str]:
134
  harness = (harness or "").lower()
135
  title = t("step_prefix")
136
  body_parts: list[str] = []
137
  action = ""
138
  url = ""
139
+ description = ""
140
+ screenshot = ""
141
 
142
  if "model_output" in step:
143
  model_output = step.get("model_output") or {}
144
+ if not isinstance(model_output, dict):
145
+ model_output = {}
146
  title = f"{t('step_prefix')} {step.get('step', step.get('metadata', {}).get('step_number', '?'))}"
147
  action = _format_browser_action(model_output)
148
+ description = _step_text(model_output.get("action_description"))
149
+ thinking = (
150
+ _step_text(model_output.get("thinking"))
151
+ or _step_text(model_output.get("thought"))
152
+ or _step_text(model_output.get("next_goal"))
153
+ )
154
  if thinking:
155
+ body_parts.append(thinking)
156
  results = step.get("result")
157
  if isinstance(results, list):
158
  for row in results[:2]:
159
  if isinstance(row, dict) and row.get("extracted_content"):
160
  body_parts.append(str(row["extracted_content"]))
161
+ url = _extract_url_from_state(step.get("state")) or _step_text(step.get("url"))
162
+ screenshot = _step_text(step.get("screenshot")) or _step_text(step.get("screenshot_path"))
163
  if not body_parts and step.get("state_message"):
164
  body_parts.append(_truncate(str(step["state_message"]), 400))
165
  elif step.get("kind") == "message":
 
179
  return {
180
  "title": title,
181
  "action": action,
182
+ "action_description": description,
183
  "body": _truncate("\n\n".join(part for part in body_parts if part), 1200),
184
  "url": url,
185
+ "screenshot": screenshot,
186
  }
187
 
188
 
189
+ def _allowed_screenshot_roots() -> list[Path]:
190
+ here = Path(__file__).resolve()
191
+ roots = [Path.cwd().resolve(), here.parents[1]]
192
+ repo = here.parents[2]
193
+ if (repo / "tests").is_dir() or (repo / "liderboard").is_dir():
194
+ roots.append(repo.resolve())
195
+ unique: list[Path] = []
196
+ for root in roots:
197
+ if root not in unique:
198
+ unique.append(root)
199
+ return unique
200
+
201
+
202
+ def _resolve_screenshot(path: str) -> Path | None:
203
+ raw = Path(path)
204
+ candidates = [raw] if raw.is_absolute() else [root / raw for root in _allowed_screenshot_roots()]
205
+ for candidate in candidates:
206
+ try:
207
+ resolved = candidate.resolve()
208
+ except OSError:
209
+ continue
210
+ if not resolved.is_file():
211
+ continue
212
+ if resolved.suffix.lower() not in {".png", ".jpg", ".jpeg", ".webp", ".gif"}:
213
+ continue
214
+ if not any(_is_relative_to(resolved, root) for root in _allowed_screenshot_roots()):
215
+ continue
216
+ return resolved
217
+ return None
218
+
219
+
220
+ def _is_relative_to(path: Path, root: Path) -> bool:
221
+ try:
222
+ path.relative_to(root)
223
+ except ValueError:
224
+ return False
225
+ return True
226
+
227
+
228
+ def _screenshot_html(path: str) -> str:
229
+ text = (path or "").strip()
230
+ if not text:
231
+ return ""
232
+ if text.startswith("data:image/"):
233
+ return (
234
+ '<img class="wab-trace-step-shot" alt="screenshot" loading="lazy" '
235
+ f'src="{html.escape(text, quote=True)}" />'
236
+ )
237
+ resolved = _resolve_screenshot(text)
238
+ if resolved is None:
239
+ return ""
240
+ src = _screenshot_data_uri(resolved)
241
+ if not src:
242
+ return ""
243
+ return (
244
+ '<img class="wab-trace-step-shot" alt="screenshot" loading="lazy" '
245
+ f'src="{src}" />'
246
+ )
247
+
248
+
249
+ def _screenshot_data_uri(path: Path) -> str:
250
+ try:
251
+ from PIL import Image
252
+
253
+ with Image.open(path) as img:
254
+ img = img.convert("RGB")
255
+ img.thumbnail((1100, 720))
256
+ buf = io.BytesIO()
257
+ img.save(buf, format="JPEG", quality=72)
258
+ encoded = base64.b64encode(buf.getvalue()).decode("ascii")
259
+ return f"data:image/jpeg;base64,{encoded}"
260
+ except Exception:
261
+ try:
262
+ data = path.read_bytes()
263
+ except OSError:
264
+ return ""
265
+ if not data or len(data) > 1_500_000:
266
+ return ""
267
+ mime = {
268
+ ".png": "image/png",
269
+ ".jpg": "image/jpeg",
270
+ ".jpeg": "image/jpeg",
271
+ ".webp": "image/webp",
272
+ ".gif": "image/gif",
273
+ }.get(path.suffix.lower(), "image/png")
274
+ encoded = base64.b64encode(data).decode("ascii")
275
+ return f"data:{mime};base64,{encoded}"
276
+
277
+
278
  def _step_sidebar_label(step: dict[str, Any], harness: str) -> str:
279
  if "model_output" in step:
280
  formatted = _format_browser_action(step.get("model_output") or {})
 
308
  "kind": "user",
309
  "title": t("task_title"),
310
  "action": "",
311
+ "action_description": "",
312
  "body": str(test.get("input") or ""),
313
  "url": str(test.get("task_url") or ""),
314
+ "screenshot": "",
315
  }
316
  ]
317
  for raw_step in agent_steps:
 
327
  "kind": _step_kind_class(raw_step),
328
  "title": norm["title"],
329
  "action": norm["action"],
330
+ "action_description": norm["action_description"],
331
  "body": norm["body"],
332
  "url": norm["url"],
333
+ "screenshot": norm["screenshot"],
334
  }
335
  )
336
  return rows
337
 
338
 
339
+ def _action_row_detail(step: dict[str, str]) -> str:
340
+ action = step.get("action") or ""
341
+ description = step.get("action_description") or ""
342
+ if description and description != action:
343
+ return f"{action}\n{description}".strip()
344
+ return action
345
+
346
+
347
  def _extract_action_rows(test: dict[str, Any], harness: str) -> list[dict[str, str]]:
348
  rows: list[dict[str, str]] = []
349
  for step in _ui_agent_steps(test, harness)[1:]:
350
+ detail = _action_row_detail(step)
351
+ if detail:
352
+ rows.append({"step": step["id"], "name": step["label"], "detail": detail})
353
  extended = test.get("extended_metrics") or {}
354
  if not rows and extended.get("action_count"):
355
  rows.append(
 
708
  step_panels: list[str] = []
709
  for step in ui_steps:
710
  active = " active" if step["id"] == "0" else ""
711
+ search_label = " ".join(
712
+ part
713
+ for part in (step["label"], step.get("action", ""), step.get("action_description", ""))
714
+ if part
715
+ )
716
  step_nav_items.append(
717
  f'<button type="button" class="wab-trace-step-item{active}" data-wab-trace-step="{html.escape(step["id"])}" '
718
+ f'data-wab-step-label="{html.escape(search_label.lower())}">'
719
  f'<span class="wab-trace-step-num">{int(step["id"]) + 1}</span>'
720
  f'<span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>'
721
  f'<span class="wab-trace-step-label">{_esc(step["label"])}</span>'
 
727
  if step["action"]
728
  else ""
729
  )
730
+ description = step.get("action_description") or ""
731
+ description_line = (
732
+ f'<div class="wab-trace-step-description">{_esc(description)}</div>'
733
+ if description and description != step.get("action")
734
+ else ""
735
+ )
736
+ body_html = f'<pre class="wab-trace-step-body">{_esc(step["body"])}</pre>' if step.get("body") else ""
737
+ shot_html = _screenshot_html(step.get("screenshot") or "")
738
  step_panels.append(
739
  f'<div class="wab-trace-step-panel{active}" data-wab-trace-step-panel="{html.escape(step["id"])}">'
740
  f'<div class="wab-trace-step-panel-head"><span class="wab-trace-step-kind wab-trace-step-kind-{html.escape(step["kind"])}"></span>'
741
  f"<strong>{_esc(step['title'])}</strong></div>"
742
+ f"{url_line}{action_line}{description_line}{shot_html}{body_html}"
 
743
  "</div>"
744
  )
745