| """A scripted computer-use agent: rebuild a task with mouse and keyboard only. |
| |
| Every edit goes through the environment's GUI tools (`left_click`, `type`, |
| `key`, `scroll`) over MCP, exactly as a model would send them. The only thing |
| the script gets for free is where controls are on screen, from a debug hook |
| that is off unless the server runs with DESIGN_CANVAS_DEBUG=1. If the GUI |
| surface is functionally accurate, typing the ground truth into the editor must |
| score (almost exactly) 1.0. |
| |
| DESIGN_CANVAS_DEBUG=1 design-canvas serve --port 8007 |
| python scripts/gui_expert.py --server http://127.0.0.1:8007 --skin canva |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import json |
| import sys |
| import time |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parents[2])) |
|
|
| from design_canvas.core.pack import load_pack |
| from design_canvas.cua import ( |
| GuiAgent as GuiExpert, |
| ) |
| from openenv.core.env_server.mcp_types import CallToolAction |
| from openenv.core.mcp_client import MCPToolClient |
|
|
|
|
| def pick_task(pack, split="test", max_elements=9, max_assets=7): |
| """A task the GUI can express exactly: no per-character runs, few assets.""" |
| for i in range(pack.count(split)): |
| t = pack.at(split, i) |
| gt = pack.gt_doc(t) |
| if (len(gt.elements) <= max_elements and len(t["assets"]) <= max_assets |
| and not any(e.runs for e in gt.elements) and t["stats"]["n_text"] >= 2): |
| yield t |
|
|
|
|
| def run(server, skin, task=None, verbose=False): |
| pack = load_pack() |
| task = task or next(pick_task(pack)) |
| gt = pack.gt_doc(task) |
| with MCPToolClient(base_url=server, message_timeout_s=120).sync() as env: |
| r = env.reset(task_id=task["task_id"], surface="gui", skin=skin, max_steps=600) |
| eid = r.observation.metadata["episode_id"] |
| bot = GuiExpert(server, env, eid, skin, verbose=verbose) |
| t0 = time.time() |
| final = bot.build(gt, {a['id']: a for a in task['assets']}) |
| grade = final.observation.metadata.get("grade") if final.done else None |
| if grade is None: |
| out = env.step(CallToolAction(tool_name="submit", arguments={})) |
| grade = out.observation.metadata.get("grade") |
| return {"task_id": task["task_id"], "skin": skin, "elements": len(gt.elements), |
| "actions": bot.actions, "seconds": round(time.time() - t0, 1), |
| "reward": grade["total"] if grade else None, "grade": grade} |
|
|
|
|
| def main(argv=None): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--server", default="http://127.0.0.1:8007") |
| from design_canvas.server.episodes import SKINS |
|
|
| ap.add_argument("--skin", default="figma", choices=list(SKINS)) |
| ap.add_argument("--task-id", default=None) |
| ap.add_argument("--verbose", action="store_true") |
| args = ap.parse_args(argv) |
| pack = load_pack() |
| task = pack.get(args.task_id) if args.task_id else None |
| res = run(args.server, args.skin, task, args.verbose) |
| print(json.dumps({k: v for k, v in res.items() if k != "grade"}, indent=2)) |
| if res["grade"]: |
| worst = sorted(res["grade"]["elements"], key=lambda e: e["score"])[:3] |
| print("weakest elements:", [(e["gt_id"], round(e["score"], 3), e["note"]) for e in worst]) |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
