| """ |
| Web Agent Trace Display Type |
| |
| Interactive step-by-step viewer for web agent browsing traces. |
| Renders screenshots with SVG overlay visualizations (click markers, |
| bounding boxes, mouse paths) alongside step details and a filmstrip navigator. |
| |
| Data format: |
| { |
| "steps": [ |
| { |
| "step_index": 0, |
| "screenshot_url": "screenshots/step_000.png", |
| "action_type": "click", |
| "element": {"tag": "input", "text": "Search", "bbox": [340, 45, 680, 75]}, |
| "coordinates": {"x": 510, "y": 60}, |
| "mouse_path": [[200, 300], [350, 200], [510, 60]], |
| "thought": "I need to search for blue wool sweaters", |
| "observation": "Search box is focused", |
| "timestamp": 1.2, |
| "viewport": {"width": 1280, "height": 720} |
| } |
| ], |
| "task_description": "Find and add a blue wool sweater under $50 to cart", |
| "site": "amazon.com" |
| } |
| |
| Supported action_type values: click, type, scroll, hover, select, navigate, wait, done |
| """ |
|
|
| import html |
| import json |
| from typing import Dict, Any, List, Optional |
|
|
| from .base import BaseDisplay |
|
|
|
|
| |
| ACTION_TYPE_COLORS = { |
| "click": {"bg": "#fff3e0", "border": "#FF9800", "badge": "rgba(255,152,0,0.2)"}, |
| "type": {"bg": "#e8f4fd", "border": "#2196F3", "badge": "rgba(33,150,243,0.2)"}, |
| "scroll": {"bg": "#e8f5e9", "border": "#4CAF50", "badge": "rgba(76,175,80,0.2)"}, |
| "hover": {"bg": "#f3e5f5", "border": "#9C27B0", "badge": "rgba(156,39,176,0.2)"}, |
| "select": {"bg": "#e0f7fa", "border": "#00BCD4", "badge": "rgba(0,188,212,0.2)"}, |
| "navigate": {"bg": "#e8eaf6", "border": "#3F51B5", "badge": "rgba(63,81,181,0.2)"}, |
| "wait": {"bg": "#f5f5f5", "border": "#9E9E9E", "badge": "rgba(158,158,158,0.2)"}, |
| "done": {"bg": "#e8f5e9", "border": "#388E3C", "badge": "rgba(56,142,60,0.2)"}, |
| } |
|
|
| DEFAULT_ACTION_COLOR = {"bg": "#f5f5f5", "border": "#9E9E9E", "badge": "rgba(158,158,158,0.2)"} |
|
|
|
|
| class WebAgentTraceDisplay(BaseDisplay): |
| """ |
| Display type for web agent browsing traces with interactive step navigation, |
| SVG overlay visualizations, and filmstrip thumbnails. |
| """ |
|
|
| name = "web_agent_trace" |
| required_fields = ["key"] |
| optional_fields = { |
| "show_overlays": True, |
| "show_filmstrip": True, |
| "show_thought": True, |
| "show_observation": True, |
| "show_element_info": True, |
| "screenshot_max_width": 800, |
| "screenshot_max_height": 600, |
| "filmstrip_size": 80, |
| "auto_playback": False, |
| "playback_step_delay": 2.0, |
| } |
| description = "Web agent trace viewer with screenshots, SVG overlays, and step navigation" |
| supports_span_target = False |
|
|
| def render(self, field_config: Dict[str, Any], data: Any) -> str: |
| if not data: |
| return '<div class="web-agent-placeholder">No trace data provided</div>' |
|
|
| options = self.get_display_options(field_config) |
| field_key = html.escape(field_config.get("key", ""), quote=True) |
|
|
| |
| steps = self._normalize_steps(data) |
| if not steps: |
| return '<div class="web-agent-placeholder">No trace steps found</div>' |
|
|
| |
| task_desc = "" |
| site = "" |
| if isinstance(data, dict): |
| task_desc = data.get("task_description", "") |
| site = data.get("site", "") |
|
|
| max_w = options.get("screenshot_max_width", 800) |
| max_h = options.get("screenshot_max_height", 600) |
| filmstrip_size = options.get("filmstrip_size", 80) |
|
|
| |
| steps_json = html.escape(json.dumps(steps, ensure_ascii=False), quote=True) |
|
|
| css = self._build_css(max_w, max_h, filmstrip_size) |
|
|
| |
| task_html = "" |
| if task_desc: |
| escaped_task = html.escape(str(task_desc)) |
| task_html = f'<div class="web-agent-task"><strong>Task:</strong> {escaped_task}</div>' |
| if site: |
| escaped_site = html.escape(str(site)) |
| task_html += f'<div class="web-agent-site"><strong>Site:</strong> {escaped_site}</div>' |
|
|
| |
| first_step = steps[0] |
| screenshot_html = self._render_screenshot(first_step, max_w, max_h) |
| details_html = self._render_step_details(first_step, 0, len(steps), options) |
| filmstrip_html = self._render_filmstrip(steps, filmstrip_size) if options.get("show_filmstrip", True) else "" |
|
|
| |
| per_step_html = '<div class="web-agent-per-step-annotations" data-step-index="0"></div>' |
|
|
| |
| auto_playback = options.get("auto_playback", False) |
| playback_delay = options.get("playback_step_delay", 2.0) |
| playback_attrs = "" |
| if auto_playback: |
| playback_attrs = f' data-auto-playback="true" data-playback-step-delay="{playback_delay}"' |
|
|
| return f''' |
| <style>{css}</style> |
| <div class="web-agent-viewer" data-field-key="{field_key}" data-steps="{steps_json}"{playback_attrs}> |
| {task_html} |
| <div class="web-agent-main"> |
| <div class="screenshot-panel"> |
| <div class="screenshot-container"> |
| <img class="step-screenshot" src="{html.escape(first_step.get('screenshot_url', ''), quote=True)}" |
| alt="Step 0 screenshot" /> |
| <svg class="overlay-layer" xmlns="http://www.w3.org/2000/svg"></svg> |
| </div> |
| <div class="step-nav"> |
| <button class="step-prev" disabled>« Prev</button> |
| <span class="step-counter">Step 1 of {len(steps)}</span> |
| <button class="step-next" {"disabled" if len(steps) <= 1 else ""}>» Next</button> |
| </div> |
| </div> |
| <div class="step-details-panel"> |
| {details_html} |
| {per_step_html} |
| </div> |
| </div> |
| {filmstrip_html} |
| <div class="overlay-controls"> |
| <label><input type="checkbox" class="overlay-toggle" data-overlay="click" checked> Clicks</label> |
| <label><input type="checkbox" class="overlay-toggle" data-overlay="bbox" checked> Bounding Boxes</label> |
| <label><input type="checkbox" class="overlay-toggle" data-overlay="path" checked> Mouse Path</label> |
| <label><input type="checkbox" class="overlay-toggle" data-overlay="scroll" checked> Scroll</label> |
| </div> |
| </div> |
| ''' |
|
|
| def _normalize_steps(self, data: Any) -> List[Dict[str, Any]]: |
| """Normalize input data to a list of step dicts.""" |
| if isinstance(data, dict): |
| steps = data.get("steps", []) |
| elif isinstance(data, list): |
| steps = data |
| else: |
| return [] |
|
|
| normalized = [] |
| for i, step in enumerate(steps): |
| if not isinstance(step, dict): |
| continue |
| normalized.append({ |
| "step_index": step.get("step_index", i), |
| "screenshot_url": step.get("screenshot_url", step.get("screenshot", "")), |
| "action_type": step.get("action_type", "unknown"), |
| "element": step.get("element", {}), |
| "coordinates": step.get("coordinates", {}), |
| "mouse_path": step.get("mouse_path", []), |
| "thought": step.get("thought", ""), |
| "observation": step.get("observation", ""), |
| "timestamp": step.get("timestamp", ""), |
| "viewport": step.get("viewport", {"width": 1280, "height": 720}), |
| "typed_text": step.get("typed_text", step.get("value", "")), |
| "scroll_direction": step.get("scroll_direction", step.get("direction", "")), |
| }) |
| return normalized |
|
|
| def _render_screenshot(self, step: Dict, max_w: int, max_h: int) -> str: |
| """Render the screenshot image element.""" |
| url = step.get("screenshot_url", "") |
| if not url: |
| return '<div class="screenshot-placeholder">No screenshot available</div>' |
| escaped_url = html.escape(str(url), quote=True) |
| return f'<img class="step-screenshot" src="{escaped_url}" alt="Step screenshot" />' |
|
|
| def _render_step_details(self, step: Dict, index: int, total: int, |
| options: Dict) -> str: |
| """Render the step details panel content.""" |
| action_type = step.get("action_type", "unknown") |
| safe_type = html.escape(str(action_type), quote=True) |
| colors = ACTION_TYPE_COLORS.get(action_type, DEFAULT_ACTION_COLOR) |
|
|
| parts = [] |
|
|
| |
| parts.append( |
| f'<div class="action-badge" style="background:{colors["badge"]}">' |
| f'{safe_type.upper()}</div>' |
| ) |
|
|
| |
| ts = step.get("timestamp", "") |
| if ts: |
| parts.append(f'<div class="step-timestamp">t={html.escape(str(ts))}s</div>') |
|
|
| |
| thought = step.get("thought", "") |
| if thought and options.get("show_thought", True): |
| parts.append( |
| f'<div class="step-thought">' |
| f'<strong>Thought:</strong> {html.escape(str(thought))}' |
| f'</div>' |
| ) |
|
|
| |
| element = step.get("element", {}) |
| if element and options.get("show_element_info", True): |
| elem_parts = [] |
| if isinstance(element, dict): |
| for k in ("tag", "text", "id", "class"): |
| if k in element: |
| elem_parts.append(f'{k}="{html.escape(str(element[k]))}"') |
| if elem_parts: |
| parts.append( |
| f'<div class="step-element">' |
| f'<strong>Element:</strong> <code>{" ".join(elem_parts)}</code>' |
| f'</div>' |
| ) |
|
|
| |
| coords = step.get("coordinates", {}) |
| if coords and isinstance(coords, dict): |
| x = coords.get("x", "") |
| y = coords.get("y", "") |
| if x or y: |
| parts.append( |
| f'<div class="step-coords">' |
| f'<strong>Coords:</strong> ({html.escape(str(x))}, {html.escape(str(y))})' |
| f'</div>' |
| ) |
|
|
| |
| typed = step.get("typed_text", "") |
| if typed: |
| parts.append( |
| f'<div class="step-typed">' |
| f'<strong>Typed:</strong> "{html.escape(str(typed))}"' |
| f'</div>' |
| ) |
|
|
| |
| obs = step.get("observation", "") |
| if obs and options.get("show_observation", True): |
| parts.append( |
| f'<div class="step-observation">' |
| f'<strong>Observation:</strong> {html.escape(str(obs))}' |
| f'</div>' |
| ) |
|
|
| return f'<div class="step-details-content">{"".join(parts)}</div>' |
|
|
| def _render_filmstrip(self, steps: List[Dict], thumb_size: int) -> str: |
| """Render the filmstrip thumbnail navigation bar.""" |
| thumbs = [] |
| for i, step in enumerate(steps): |
| url = step.get("screenshot_url", "") |
| active = "filmstrip-active" if i == 0 else "" |
| if url: |
| escaped_url = html.escape(str(url), quote=True) |
| thumbs.append( |
| f'<div class="filmstrip-thumb {active}" data-step="{i}">' |
| f'<img src="{escaped_url}" alt="Step {i}" />' |
| f'<span class="filmstrip-label">{i + 1}</span>' |
| f'</div>' |
| ) |
| else: |
| thumbs.append( |
| f'<div class="filmstrip-thumb {active}" data-step="{i}">' |
| f'<div class="filmstrip-placeholder">?</div>' |
| f'<span class="filmstrip-label">{i + 1}</span>' |
| f'</div>' |
| ) |
| return f'<div class="filmstrip">{"".join(thumbs)}</div>' |
|
|
| def _build_css(self, max_w: int, max_h: int, filmstrip_size: int) -> str: |
| """Build CSS for the web agent viewer.""" |
| return f''' |
| .web-agent-viewer {{ font-family: inherit; }} |
| .web-agent-task, .web-agent-site {{ |
| padding: 6px 12px; margin-bottom: 8px; |
| background: #f8f9fa; border-radius: 4px; font-size: 0.95em; |
| }} |
| .web-agent-main {{ |
| display: flex; gap: 16px; margin-bottom: 12px; |
| }} |
| .screenshot-panel {{ |
| flex: 0 0 auto; max-width: {max_w}px; |
| }} |
| .screenshot-container {{ |
| position: relative; display: inline-block; |
| border: 1px solid #ddd; border-radius: 6px; overflow: hidden; |
| background: #1a1a1a; |
| }} |
| .step-screenshot {{ |
| display: block; max-width: {max_w}px; max-height: {max_h}px; |
| width: auto; height: auto; |
| }} |
| .overlay-layer {{ |
| position: absolute; top: 0; left: 0; |
| width: 100%; height: 100%; |
| pointer-events: none; |
| }} |
| .screenshot-placeholder {{ |
| width: {max_w}px; height: 300px; |
| display: flex; align-items: center; justify-content: center; |
| background: #f0f0f0; color: #999; font-size: 0.9em; |
| }} |
| .step-nav {{ |
| display: flex; align-items: center; justify-content: center; |
| gap: 12px; padding: 8px 0; |
| }} |
| .step-nav button {{ |
| padding: 4px 12px; border: 1px solid #ccc; border-radius: 4px; |
| background: #fff; cursor: pointer; font-size: 0.85em; |
| }} |
| .step-nav button:disabled {{ |
| opacity: 0.4; cursor: default; |
| }} |
| .step-nav button:not(:disabled):hover {{ |
| background: #e8f4fd; border-color: #2196F3; |
| }} |
| .step-counter {{ |
| font-size: 0.9em; font-weight: 600; color: #555; |
| }} |
| .step-details-panel {{ |
| flex: 1; min-width: 250px; max-width: 400px; |
| border: 1px solid #e0e0e0; border-radius: 6px; |
| padding: 12px; background: #fafafa; overflow-y: auto; |
| max-height: {max_h + 50}px; |
| }} |
| .step-details-content {{ |
| display: flex; flex-direction: column; gap: 8px; |
| }} |
| .action-badge {{ |
| display: inline-block; padding: 4px 12px; |
| border-radius: 4px; font-weight: 700; |
| font-size: 0.8em; letter-spacing: 0.5px; |
| }} |
| .step-timestamp {{ color: #888; font-size: 0.8em; }} |
| .step-thought {{ |
| padding: 8px; background: #e8f4fd; |
| border-left: 3px solid #2196F3; border-radius: 4px; |
| font-size: 0.9em; |
| }} |
| .step-element {{ |
| font-size: 0.85em; color: #555; |
| }} |
| .step-element code {{ |
| background: #f0f0f0; padding: 2px 4px; border-radius: 2px; |
| font-size: 0.9em; |
| }} |
| .step-coords {{ font-size: 0.85em; color: #666; }} |
| .step-typed {{ |
| padding: 6px 8px; background: #e8f4fd; |
| border-radius: 4px; font-size: 0.9em; |
| }} |
| .step-observation {{ |
| padding: 8px; background: #e8f5e9; |
| border-left: 3px solid #4CAF50; border-radius: 4px; |
| font-size: 0.9em; |
| }} |
| .filmstrip {{ |
| display: flex; gap: 4px; overflow-x: auto; |
| padding: 8px 4px; background: #f5f5f5; |
| border-radius: 6px; margin-top: 4px; |
| }} |
| .filmstrip-thumb {{ |
| flex: 0 0 auto; width: {filmstrip_size}px; |
| cursor: pointer; border: 2px solid transparent; |
| border-radius: 4px; overflow: hidden; |
| text-align: center; background: #fff; |
| transition: border-color 0.2s; |
| }} |
| .filmstrip-thumb:hover {{ border-color: #90CAF9; }} |
| .filmstrip-thumb.filmstrip-active {{ border-color: #2196F3; }} |
| .filmstrip-thumb img {{ |
| width: 100%; height: {int(filmstrip_size * 0.65)}px; |
| object-fit: cover; display: block; |
| }} |
| .filmstrip-placeholder {{ |
| width: 100%; height: {int(filmstrip_size * 0.65)}px; |
| display: flex; align-items: center; justify-content: center; |
| background: #eee; color: #999; font-size: 0.8em; |
| }} |
| .filmstrip-label {{ |
| font-size: 0.7em; color: #666; padding: 2px 0; |
| }} |
| .overlay-controls {{ |
| display: flex; gap: 12px; padding: 6px 0; |
| font-size: 0.8em; color: #555; |
| }} |
| .overlay-controls label {{ |
| display: flex; align-items: center; gap: 4px; cursor: pointer; |
| }} |
| .web-agent-per-step-annotations {{ |
| margin-top: 12px; padding-top: 12px; |
| border-top: 1px solid #e0e0e0; |
| }} |
| |
| /* SVG overlay styles */ |
| .overlay-click-marker {{ }} |
| .overlay-bbox {{ }} |
| .overlay-mouse-path {{ }} |
| .overlay-scroll {{ }} |
| |
| @keyframes pulse-marker {{ |
| 0%, 100% {{ r: 8; opacity: 1; }} |
| 50% {{ r: 12; opacity: 0.7; }} |
| }} |
| |
| @media (max-width: 768px) {{ |
| .web-agent-main {{ flex-direction: column; }} |
| .step-details-panel {{ max-width: none; max-height: none; }} |
| .screenshot-panel {{ max-width: 100%; }} |
| .step-screenshot {{ max-width: 100%; }} |
| }} |
| ''' |
|
|
| def get_css_classes(self, field_config: Dict[str, Any]) -> List[str]: |
| classes = super().get_css_classes(field_config) |
| return classes |
|
|
| def get_data_attributes(self, field_config: Dict[str, Any], data: Any) -> Dict[str, str]: |
| attrs = super().get_data_attributes(field_config, data) |
| return attrs |
|
|
