""" Live Agent Display Type Dual-mode display: - When data is empty/None: renders the live agent UI (SSE-connected viewer, controls, instruction input, thought panel, filmstrip) - When data contains a completed trace: delegates to WebAgentTraceDisplay for post-hoc review Configuration example: instance_display: fields: - key: agent_trace type: live_agent label: "Live Agent Session" display_options: show_overlays: true show_filmstrip: true show_thought: true show_controls: true allow_takeover: true allow_instructions: true screenshot_max_width: 900 screenshot_max_height: 650 """ import html import json from typing import Any, Dict, List from .base import BaseDisplay class LiveAgentDisplay(BaseDisplay): """ Display type for live AI agent interaction. Renders a viewer that connects to the agent SSE stream, showing real-time screenshots, overlay visualizations, and providing controls for pause/resume/instruct/takeover. """ name = "live_agent" required_fields = ["key"] optional_fields = { "show_overlays": True, "show_filmstrip": True, "show_thought": True, "show_controls": True, "allow_takeover": True, "allow_instructions": True, "screenshot_max_width": 900, "screenshot_max_height": 650, "filmstrip_size": 80, } description = "Live AI agent viewer with real-time screenshots, controls, and interaction" supports_span_target = False def render(self, field_config: Dict[str, Any], data: Any) -> str: """ Render live agent UI or delegate to trace viewer. If data contains a completed trace (dict with 'steps'), delegates to WebAgentTraceDisplay. Otherwise renders the live interaction UI. """ # Check if data is a completed trace (has steps) if data and isinstance(data, dict) and data.get("steps"): return self._render_review_mode(field_config, data) return self._render_live_mode(field_config, data) def _render_review_mode(self, field_config: Dict[str, Any], data: dict) -> str: """Delegate to WebAgentTraceDisplay for completed traces.""" from .web_agent_trace_display import WebAgentTraceDisplay reviewer = WebAgentTraceDisplay() return reviewer.render(field_config, data) def _render_live_mode(self, field_config: Dict[str, Any], data: Any) -> str: """Render the live agent interaction UI.""" options = self.get_display_options(field_config) field_key = html.escape(field_config.get("key", ""), quote=True) max_w = options.get("screenshot_max_width", 900) max_h = options.get("screenshot_max_height", 650) filmstrip_size = options.get("filmstrip_size", 80) show_controls = options.get("show_controls", True) show_thought = options.get("show_thought", True) show_filmstrip = options.get("show_filmstrip", True) show_overlays = options.get("show_overlays", True) allow_takeover = options.get("allow_takeover", True) allow_instructions = options.get("allow_instructions", True) # Encode config as data attributes for JS config_json = html.escape(json.dumps({ "show_overlays": show_overlays, "show_filmstrip": show_filmstrip, "show_thought": show_thought, "show_controls": show_controls, "allow_takeover": allow_takeover, "allow_instructions": allow_instructions, }), quote=True) # Extract task info from instance data if available task_desc = "" start_url = "" if isinstance(data, dict): task_desc = data.get("task_description", "") start_url = data.get("start_url", data.get("url", "")) css = self._build_css(max_w, max_h, filmstrip_size) html_content = self._build_html( field_key, config_json, task_desc, start_url, show_controls, show_thought, show_filmstrip, show_overlays, allow_takeover, allow_instructions, max_w, max_h, filmstrip_size, ) return f"\n{html_content}" def _build_css(self, max_w: int, max_h: int, filmstrip_size: int) -> str: return f""" .live-agent-viewer {{ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; border: 1px solid #ddd; border-radius: 8px; overflow: hidden; background: #fafafa; }} /* Status bar */ .live-agent-status {{ display: flex; align-items: center; justify-content: space-between; padding: 8px 16px; background: #f0f0f0; border-bottom: 1px solid #ddd; font-size: 13px; }} .live-agent-status-indicator {{ display: inline-block; width: 10px; height: 10px; border-radius: 50%; margin-right: 8px; }} .live-agent-status-indicator.idle {{ background: #9E9E9E; }} .live-agent-status-indicator.running {{ background: #4CAF50; animation: pulse-dot 1.5s infinite; }} .live-agent-status-indicator.paused {{ background: #FF9800; }} .live-agent-status-indicator.takeover {{ background: #2196F3; }} .live-agent-status-indicator.completed {{ background: #388E3C; }} .live-agent-status-indicator.error {{ background: #F44336; }} @keyframes pulse-dot {{ 0%, 100% {{ opacity: 1; }} 50% {{ opacity: 0.5; }} }} /* Main layout */ .live-agent-main {{ display: flex; gap: 0; }} .live-agent-screenshot-panel {{ flex: 1; position: relative; background: #000; min-height: 200px; max-width: {max_w}px; }} .live-agent-screenshot {{ width: 100%; max-height: {max_h}px; object-fit: contain; display: block; }} /* Collapsed state — shrink screenshot for more annotation space */ .live-agent-viewer.collapsed .live-agent-main {{ max-height: 220px; overflow: hidden; }} .live-agent-viewer.collapsed .live-agent-screenshot {{ max-height: 200px; }} .live-agent-viewer.collapsed .live-agent-side-panel {{ display: none; }} /* Collapse toggle */ .live-agent-collapse-toggle {{ cursor: pointer; font-size: 12px; color: #666; padding: 2px 8px; border: 1px solid #ccc; border-radius: 4px; background: #fff; }} .live-agent-collapse-toggle:hover {{ background: #f0f0f0; }} /* Takeover click feedback */ .live-agent-click-marker {{ position: absolute; width: 24px; height: 24px; border: 2px solid #F44336; border-radius: 50%; background: rgba(244, 67, 54, 0.2); transform: translate(-50%, -50%); pointer-events: none; animation: click-pulse 0.6s ease-out forwards; z-index: 100; }} @keyframes click-pulse {{ 0% {{ transform: translate(-50%, -50%) scale(0.5); opacity: 1; }} 100% {{ transform: translate(-50%, -50%) scale(2); opacity: 0; }} }} /* Takeover action toast */ .live-agent-action-toast {{ position: absolute; bottom: 8px; left: 50%; transform: translateX(-50%); background: rgba(0,0,0,0.8); color: #fff; padding: 4px 12px; border-radius: 4px; font-size: 12px; pointer-events: none; z-index: 100; animation: toast-fade 1.5s ease-out forwards; }} @keyframes toast-fade {{ 0% {{ opacity: 1; }} 70% {{ opacity: 1; }} 100% {{ opacity: 0; }} }} .live-agent-screenshot-placeholder {{ display: flex; align-items: center; justify-content: center; min-height: 400px; color: #888; font-size: 16px; }} .live-agent-overlay-layer {{ position: absolute; top: 0; left: 0; width: 100%; height: 100%; pointer-events: none; }} /* Side panel */ .live-agent-side-panel {{ width: 320px; display: flex; flex-direction: column; border-left: 1px solid #ddd; background: #fff; }} /* Thought panel */ .live-agent-thought-panel {{ padding: 12px; border-bottom: 1px solid #eee; max-height: 200px; overflow-y: auto; }} .live-agent-thought-panel h4 {{ margin: 0 0 8px 0; font-size: 12px; text-transform: uppercase; color: #666; }} .live-agent-thought-text {{ font-size: 13px; line-height: 1.5; color: #333; white-space: pre-wrap; }} /* Step details */ .live-agent-step-details {{ padding: 12px; flex: 1; overflow-y: auto; font-size: 13px; }} .live-agent-action-badge {{ display: inline-block; padding: 2px 8px; border-radius: 4px; font-size: 12px; font-weight: 600; text-transform: uppercase; }} /* Controls */ .live-agent-controls {{ padding: 12px; border-top: 1px solid #ddd; background: #f8f8f8; }} .live-agent-control-buttons {{ display: flex; gap: 8px; flex-wrap: wrap; margin-bottom: 8px; }} .live-agent-btn {{ padding: 6px 14px; border: 1px solid #ddd; border-radius: 4px; background: #fff; cursor: pointer; font-size: 13px; transition: all 0.15s; }} .live-agent-btn:hover {{ background: #f0f0f0; }} .live-agent-btn.primary {{ background: #2196F3; color: #fff; border-color: #1976D2; }} .live-agent-btn.primary:hover {{ background: #1976D2; }} .live-agent-btn.danger {{ background: #F44336; color: #fff; border-color: #D32F2F; }} .live-agent-btn.danger:hover {{ background: #D32F2F; }} .live-agent-btn.warning {{ background: #FF9800; color: #fff; border-color: #F57C00; }} .live-agent-btn.warning:hover {{ background: #F57C00; }} .live-agent-btn.active {{ background: #4CAF50; color: #fff; border-color: #388E3C; }} .live-agent-btn:disabled {{ opacity: 0.5; cursor: not-allowed; }} /* Instruction input */ .live-agent-instruction-input {{ display: flex; gap: 8px; margin-top: 8px; }} .live-agent-instruction-input input {{ flex: 1; padding: 6px 10px; border: 1px solid #ddd; border-radius: 4px; font-size: 13px; }} /* Start form */ .live-agent-start-form {{ padding: 24px; text-align: center; }} .live-agent-start-form input {{ display: block; width: 100%; max-width: 500px; margin: 8px auto; padding: 8px 12px; border: 1px solid #ddd; border-radius: 4px; font-size: 14px; }} /* Filmstrip */ .live-agent-filmstrip {{ display: flex; gap: 4px; padding: 8px; overflow-x: auto; border-top: 1px solid #ddd; background: #f5f5f5; }} .live-agent-filmstrip-thumb {{ width: {filmstrip_size}px; height: {int(filmstrip_size * 0.75)}px; object-fit: cover; border: 2px solid transparent; border-radius: 4px; cursor: pointer; flex-shrink: 0; opacity: 0.7; transition: all 0.15s; }} .live-agent-filmstrip-thumb:hover {{ opacity: 1; }} .live-agent-filmstrip-thumb.active {{ border-color: #2196F3; opacity: 1; }} /* Takeover cursor */ .live-agent-screenshot-panel.takeover-mode {{ cursor: crosshair; }} .live-agent-screenshot-panel.takeover-mode .live-agent-overlay-layer {{ pointer-events: auto; }} /* Overlay controls */ .live-agent-overlay-controls {{ display: flex; gap: 12px; padding: 6px 12px; background: rgba(0,0,0,0.03); border-top: 1px solid #eee; font-size: 12px; }} .live-agent-overlay-controls label {{ display: flex; align-items: center; gap: 4px; cursor: pointer; }} """ def _build_html( self, field_key, config_json, task_desc, start_url, show_controls, show_thought, show_filmstrip, show_overlays, allow_takeover, allow_instructions, max_w, max_h, filmstrip_size, ) -> str: # Build the UI components parts = [] # Main container parts.append( f'
' ) # Status bar parts.append("""
Ready Step 0
""") # Start form (shown when no session is active) task_desc_escaped = html.escape(task_desc, quote=True) start_url_escaped = html.escape(start_url, quote=True) parts.append(f"""

Start Agent Session

""") # Main viewer (hidden until session starts) parts.append('') # main # Overlay controls if show_overlays: parts.append("""
""") # Filmstrip if show_filmstrip: parts.append( '' ) parts.append('
') # viewer return "\n".join(parts)