File size: 3,473 Bytes
df84e45 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 | """A scripted computer-use agent: rebuild a task with mouse and keyboard only.
Every edit goes through the environment's GUI tools (`left_click`, `type`,
`key`, `scroll`) over MCP, exactly as a model would send them. The only thing
the script gets for free is where controls are on screen, from a debug hook
that is off unless the server runs with DESIGN_CANVAS_DEBUG=1. If the GUI
surface is functionally accurate, typing the ground truth into the editor must
score (almost exactly) 1.0.
DESIGN_CANVAS_DEBUG=1 design-canvas serve --port 8007
python scripts/gui_expert.py --server http://127.0.0.1:8007 --skin canva
"""
from __future__ import annotations
import argparse
import json
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from design_canvas.core.pack import load_pack # noqa: E402
from design_canvas.cua import (
GuiAgent as GuiExpert, # noqa: E402 (the plain, fixed-behaviour agent)
)
from openenv.core.env_server.mcp_types import CallToolAction # noqa: E402
from openenv.core.mcp_client import MCPToolClient # noqa: E402
def pick_task(pack, split="test", max_elements=9, max_assets=7):
"""A task the GUI can express exactly: no per-character runs, few assets."""
for i in range(pack.count(split)):
t = pack.at(split, i)
gt = pack.gt_doc(t)
if (len(gt.elements) <= max_elements and len(t["assets"]) <= max_assets
and not any(e.runs for e in gt.elements) and t["stats"]["n_text"] >= 2):
yield t
def run(server, skin, task=None, verbose=False):
pack = load_pack()
task = task or next(pick_task(pack))
gt = pack.gt_doc(task)
with MCPToolClient(base_url=server, message_timeout_s=120).sync() as env:
r = env.reset(task_id=task["task_id"], surface="gui", skin=skin, max_steps=600)
eid = r.observation.metadata["episode_id"]
bot = GuiExpert(server, env, eid, skin, verbose=verbose)
t0 = time.time()
final = bot.build(gt, {a['id']: a for a in task['assets']})
grade = final.observation.metadata.get("grade") if final.done else None
if grade is None: # submit landed on a later settle; ask for the state
out = env.step(CallToolAction(tool_name="submit", arguments={}))
grade = out.observation.metadata.get("grade")
return {"task_id": task["task_id"], "skin": skin, "elements": len(gt.elements),
"actions": bot.actions, "seconds": round(time.time() - t0, 1),
"reward": grade["total"] if grade else None, "grade": grade}
def main(argv=None):
ap = argparse.ArgumentParser()
ap.add_argument("--server", default="http://127.0.0.1:8007")
from design_canvas.server.episodes import SKINS
ap.add_argument("--skin", default="figma", choices=list(SKINS))
ap.add_argument("--task-id", default=None)
ap.add_argument("--verbose", action="store_true")
args = ap.parse_args(argv)
pack = load_pack()
task = pack.get(args.task_id) if args.task_id else None
res = run(args.server, args.skin, task, args.verbose)
print(json.dumps({k: v for k, v in res.items() if k != "grade"}, indent=2))
if res["grade"]:
worst = sorted(res["grade"]["elements"], key=lambda e: e["score"])[:3]
print("weakest elements:", [(e["gt_id"], round(e["score"], 3), e["note"]) for e in worst])
return 0
if __name__ == "__main__":
raise SystemExit(main())
|