"""A scripted computer-use agent: rebuild a task with mouse and keyboard only. Every edit goes through the environment's GUI tools (`left_click`, `type`, `key`, `scroll`) over MCP, exactly as a model would send them. The only thing the script gets for free is where controls are on screen, from a debug hook that is off unless the server runs with DESIGN_CANVAS_DEBUG=1. If the GUI surface is functionally accurate, typing the ground truth into the editor must score (almost exactly) 1.0. DESIGN_CANVAS_DEBUG=1 design-canvas serve --port 8007 python scripts/gui_expert.py --server http://127.0.0.1:8007 --skin canva """ from __future__ import annotations import argparse import json import sys import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from design_canvas.core.pack import load_pack # noqa: E402 from design_canvas.cua import ( GuiAgent as GuiExpert, # noqa: E402 (the plain, fixed-behaviour agent) ) from openenv.core.env_server.mcp_types import CallToolAction # noqa: E402 from openenv.core.mcp_client import MCPToolClient # noqa: E402 def pick_task(pack, split="test", max_elements=9, max_assets=7): """A task the GUI can express exactly: no per-character runs, few assets.""" for i in range(pack.count(split)): t = pack.at(split, i) gt = pack.gt_doc(t) if (len(gt.elements) <= max_elements and len(t["assets"]) <= max_assets and not any(e.runs for e in gt.elements) and t["stats"]["n_text"] >= 2): yield t def run(server, skin, task=None, verbose=False): pack = load_pack() task = task or next(pick_task(pack)) gt = pack.gt_doc(task) with MCPToolClient(base_url=server, message_timeout_s=120).sync() as env: r = env.reset(task_id=task["task_id"], surface="gui", skin=skin, max_steps=600) eid = r.observation.metadata["episode_id"] bot = GuiExpert(server, env, eid, skin, verbose=verbose) t0 = time.time() final = bot.build(gt, {a['id']: a for a in task['assets']}) grade = final.observation.metadata.get("grade") if final.done else None if grade is None: # submit landed on a later settle; ask for the state out = env.step(CallToolAction(tool_name="submit", arguments={})) grade = out.observation.metadata.get("grade") return {"task_id": task["task_id"], "skin": skin, "elements": len(gt.elements), "actions": bot.actions, "seconds": round(time.time() - t0, 1), "reward": grade["total"] if grade else None, "grade": grade} def main(argv=None): ap = argparse.ArgumentParser() ap.add_argument("--server", default="http://127.0.0.1:8007") from design_canvas.server.episodes import SKINS ap.add_argument("--skin", default="figma", choices=list(SKINS)) ap.add_argument("--task-id", default=None) ap.add_argument("--verbose", action="store_true") args = ap.parse_args(argv) pack = load_pack() task = pack.get(args.task_id) if args.task_id else None res = run(args.server, args.skin, task, args.verbose) print(json.dumps({k: v for k, v in res.items() if k != "grade"}, indent=2)) if res["grade"]: worst = sorted(res["grade"]["elements"], key=lambda e: e["score"])[:3] print("weakest elements:", [(e["gt_id"], round(e["score"], 3), e["note"]) for e in worst]) return 0 if __name__ == "__main__": raise SystemExit(main())