"""LLM-driven browser agent — the 'dynamic/unknown page' pattern. Replaces brittle selector scripts with a reason-act loop: read a sanitized page state, let the model choose the next tool call, execute it, repeat. Self-healing (re-reasons each step), capped (hard step limit = budget guardrail), and fully traced for audit. The 'brain' is the cost-aware router, so each step is routed and metered like any other LLM call. Offline, the deterministic provider replays a recorded plan so the entire loop — tool-calling, page-state sanitization, trace, metrics — runs with zero deps. With a frontier model + Playwright configured, the same loop drives a live browser. """ from __future__ import annotations import json import re import uuid from ..config import Settings from ..metrics import MetricsStore from ..prompts import BROWSER_AGENT_SYSTEM from ..providers import CacheBlock, LLMRequest from ..router import ModelRouter from ..tools import browser_registry from .session import get_session MAX_STEPS = 8 def _tool_defs_block(registry) -> str: lines = ["Tool definitions (JSON-schema):"] for d in registry.definitions(): lines.append(f"- {d['name']}: {d['description']} params={json.dumps(d['parameters'])}") return "\n".join(lines) def _parse_decision(text: str) -> dict: text = (text or "").strip() text = re.sub(r"^```(?:json)?", "", text).strip() text = re.sub(r"```$", "", text).strip() try: return json.loads(text) except json.JSONDecodeError: m = re.search(r"\{.*\}", text, re.DOTALL) if m: try: return json.loads(m.group(0)) except json.JSONDecodeError: return {} return {} def _build_plan(scenario: str, base_url: str, order: dict | None) -> list[dict]: """The recorded plan the offline provider replays (ignored by real LLMs).""" if scenario == "complex_order": # intricate multi-step interaction: dashboard → Procurement → +Create Order # → read the complex order-form fields. return [ {"tool": "navigate", "args": {"url": f"{base_url}/erp/"}, "reason": "Open the ERP dashboard."}, {"tool": "click", "args": {"selector": "#tile-procurement"}, "reason": "Click the Procurement module tile."}, {"tool": "click", "args": {"selector": "#create-order"}, "reason": "Click '+ Create Order' to open the order-form modal."}, {"tool": "extract", "args": {}, "reason": "Read the complex order fields (vendor, terms, ship-to, line items, totals, approver)."}, {"tool": "done", "args": {"result": None}, "reason": "Captured the order; finish."}, ] if scenario == "order_fill": order = order or {} plan = [{"tool": "navigate", "args": {"url": f"{base_url}/orders/new"}, "reason": "Open the new-order form."}] field_map = {"vendor_id": "#vendor-id", "sku": "#sku", "quantity": "#qty", "delivery_date": "#delivery-date"} for k, sel in field_map.items(): if order.get(k) is not None: plan.append({"tool": "fill", "args": {"selector": sel, "text": str(order[k])}, "reason": f"Fill {k} from the order data."}) plan += [ {"tool": "click", "args": {"selector": "#submit-order"}, "reason": "Submit the order."}, {"tool": "extract", "args": {}, "reason": "Capture the confirmation id."}, {"tool": "done", "args": {"result": None}, "reason": "Order submitted; return result."}, ] return plan # default: scrape pending orders return [ {"tool": "navigate", "args": {"url": f"{base_url}/orders"}, "reason": "Navigate to the pending-orders list."}, {"tool": "extract", "args": {}, "reason": "Read the orders table as structured JSON."}, {"tool": "done", "args": {"result": None}, "reason": "Have the data; finish."}, ] def run_browser_agent( goal: str, *, router: ModelRouter, settings: Settings, metrics: MetricsStore, scenario: str = "scrape_orders", order: dict | None = None, base_url: str | None = None, headless: bool = True, prefer_simulated: bool | None = None, ) -> dict: base_url = base_url or settings.demo_portal_url run_id = uuid.uuid4().hex session = get_session(headless=headless, prefer_simulated=prefer_simulated) registry = browser_registry() registry.bind("navigate", lambda url: session.navigate(url)) registry.bind("click", lambda selector: session.click(selector)) registry.bind("fill", lambda selector, text: session.fill(selector, text)) registry.bind("extract", lambda: session.extract()) registry.bind("screenshot", lambda: session.screenshot()) plan = _build_plan(scenario, base_url, order) tool_defs = _tool_defs_block(registry) # Decision "brain": a capable frontier agent LLM (Claude/Gemini) drives autonomously # when configured; otherwise we replay a recorded plan deterministically (reliable RPA). # (MiniCPM-V is used for OCR/extraction, not as the browser-agent controller.) reg = router.registry if reg.anthropic and reg.anthropic.available(): agent_provider, agent_model, agent_mode = reg.anthropic, settings.anthropic_model_smart, "llm:claude" elif reg.gemini and reg.gemini.available(): agent_provider, agent_model, agent_mode = reg.gemini, settings.gemini_model, "llm:gemini" else: agent_provider, agent_model, agent_mode = reg.mock, "mock", "deterministic-plan" trace: list[dict] = [] final_result = None last_extract = None for step in range(MAX_STEPS): page_state = session.get_state() req = LLMRequest( system_blocks=[ CacheBlock(BROWSER_AGENT_SYSTEM, cacheable=True), CacheBlock(tool_defs, cacheable=True), ], user_content=f"GOAL: {goal}\n\nCURRENT PAGE:\n{page_state}", task="agent", max_tokens=400, context={"plan": plan, "step": step}, ) resp = agent_provider.complete(req, agent_model) metrics.record_call(run_id, resp, "agent") decision = _parse_decision(resp.text) tool = decision.get("tool", "done") args = decision.get("args", {}) or {} reason = decision.get("reason", "") entry = { "step": step + 1, "tool": tool, "args": args, "reason": reason, "model": resp.model, "page_excerpt": page_state[:240], } if tool == "done": result = args.get("result") if result in (None, "", "no further actions") and last_extract is not None: result = last_extract final_result = result entry["note"] = "agent finished" trace.append(entry) break try: out = registry.call(tool, args) if tool == "extract": last_extract = out entry["note"] = "extracted: " + json.dumps(out)[:200] else: entry["note"] = getattr(out, "note", str(out))[:200] except Exception as e: entry["note"] = f"tool error: {e}" trace.append(entry) if hasattr(session, "close"): session.close() agg = metrics.call_aggregates(run_id) return { "mode": "agentic", "agent_mode": agent_mode, "backend": session.backend, "goal": goal, "scenario": scenario, "steps": len(trace), "trace": trace, "result": final_result, "run_id": run_id, "tokens": agg["input_tokens"] + agg["output_tokens"], "cost_usd": agg["cost_usd"], "cache_hits": agg["cache_hits"], }