| """Computer-use (CUA) trajectories: rebuild a design with mouse and keyboard, recorded for SFT. |
| |
| A scripted agent drives the GUI surface through the environment's own tools |
| (`left_click`, `type`, `key`, `left_click_drag`, `scroll` over MCP), exactly as |
| a model would, and each turn is recorded the way the rollout harness presents |
| it to a model: a short thought, one or a few tool calls, then a fresh |
| screenshot. The recording is only kept if the submitted design scores 1.0, so |
| every trajectory is a verified solution. |
| |
| It is built to look like a competent agent, not a macro: |
| |
| - **Grounded.** Every coordinate is where the control is on the *current* |
| screen. The only thing the script gets for free is that lookup (a debug |
| hook, `DESIGN_CANVAS_DEBUG=1`); the recording keeps only the coordinates, so |
| a model trained on it has to find controls from pixels. |
| - **Looks, then acts.** One turn = one intention: open a panel, place an asset, |
| set a field (click, select, type, commit). A screenshot follows every turn, |
| and a field below the fold is scrolled into view first. |
| - **Images come from files.** Each layout's own route opens the computer's file |
| window (Studio's Shape tools > Image/video..., Pixel's File > Place Embedded..., |
| Deck's Insert > Upload from computer, Present's Insert > Pictures > This Device..., |
| Vector's File > Import..., Template's Uploads > Upload files). The file is then |
| opened (Open, double-click or Enter) or dragged out of the window onto the |
| canvas, moving or shrinking the window first when it covers the spot. Studio |
| then places by click or by dragging out the box; Vector asks how to import; |
| Template lists uploads, which are clicked in or dragged to place. |
| - **Varied gestures, seeded.** Text is placed by clicking near its spot; fields are selected by |
| triple-click or ctrl+a and committed by enter or tab; property order varies; |
| fonts are searched or picked from "Fonts in this design". |
| - **Mistakes and recovery.** A few turns contain a realistic slip (a typo in a |
| field, the wrong asset) followed by noticing and fixing it. The slip turn is |
| marked `"train": false` so a trainer can mask it and learn only the recovery. |
| - **Every look.** All three layouts (Studio, Template, Pixel) in light and dark, |
| dealt out evenly across a batch. The target is never visible in the editor; |
| the agent has it in its first message and can call `view_reference`. |
| |
| Same task + seed gives the same trajectory. |
| |
| design-canvas export-cua --split train --limit 200 --out artifacts/cua |
| """ |
|
|
| from __future__ import annotations |
|
|
| import base64 |
| import hashlib |
| import io |
| import json |
| import random |
| import threading |
| import time |
| import urllib.request |
| from dataclasses import dataclass, field |
| from pathlib import Path |
|
|
| from openenv.core.env_server.mcp_types import CallToolAction |
|
|
| from .server.environment import FILE_DIALOGS |
| from .server.episodes import SKINS |
|
|
| VIEW_W, VIEW_H = 1280, 800 |
| THEMES = ("light", "dark") |
| LAYOUTS = ("default",) |
| RECORDER_VERSION = 3 |
|
|
| |
| |
| OPENERS = { |
| "assets": {"canva": '.cv-rail-btn[data-panel="uploads"]'}, |
| "inspector": {"figma": '.fg-tabs [data-tab="design"]', |
| "canva": [('.cv-context [data-open="position"]', '.cv-editpanel[data-panel="position"]'), |
| ('.cv-pos-tabs [data-tab="arrange"]', '.cv-pos-tabs [data-tab="arrange"].on')], |
| "photoshop": '.ps-tabs [data-tab="properties"]', "slides": '.sl-toolbar [data-open="format"]', |
| "powerpoint": [('.pp-tabs [data-tab="format"]', '.pp-tabs [data-tab="format"].on'), |
| ('.pp-ribbon [data-open="format"]', None)], |
| "inkscape": '[data-command="toolMove"]'}, |
| } |
| STEP_NAME = {'.pp-tabs [data-tab="insert"]': "the Insert tab", '.pp-tabs [data-tab="home"]': "the Home tab", |
| '.pp-tabs [data-tab="format"]': "the Shape Format tab", '.pp-tabs [data-tab="design"]': "the Design tab", |
| '.cv-context [data-open="position"]': "Position in the toolbar", '.cv-pos-tabs [data-tab="arrange"]': "the Arrange tab"} |
| |
| BG_ROUTES = {"canva": ('.cv-rail-btn[data-panel="background"]', '.cv-rail-btn[data-panel="background"].on', |
| ".cv-bgpanel .cv-bgpalette"), |
| |
| "figma": ('.layers .fg-frame-row', '.inspector .fg-frame-head', '.inspector .fg-frame [data-field="background"] .hex-input'), |
| |
| "inkscape": ('[data-command="dlg_document"]', '.ik-docktab.on[data-tab="document"]', ".ik-dp .ik-bgchip"), |
| |
| "slides": (".sl-toolbar .sl-bgbtn", ".modal.sl-bg", ".modal.sl-bg .sl-bgchip")} |
| BG_NAME = {"canva": "the Background tab", "figma": "Frame 1 in the layers", "inkscape": "Document Properties", |
| "slides": "the Background button"} |
| ACTIVE = { |
| "assets": {"canva": '.cv-rail-btn[data-panel="uploads"].on'}, |
| "inspector": {"canva": '.cv-editpanel[data-panel="position"] .cv-pos-tabs [data-tab="arrange"].on', |
| "figma": '.fg-tabs [data-tab="design"].on', |
| "photoshop": '.ps-tabs [data-tab="properties"].on', "slides": '.sl-side[data-panel="format"]', |
| "powerpoint": '.pp-pane[data-panel="format"]', "inkscape": '.ik-tool.on[data-command="toolMove"]'}, |
| } |
| PANEL_NAME = {"assets": {"canva": "Uploads"}, |
| "inspector": {"figma": "Design", "canva": "Position", "photoshop": "Properties", "slides": "Format options", |
| "powerpoint": "Format", "inkscape": "Selector"}} |
| |
| |
| |
| PLACE_ITEM = '.menu [data-command="placeImage"]' |
| FILE_ROUTES = { |
| "figma": [[('.fg-caret[title="Shape tools"]', None, None, "the Shape tools menu"), (PLACE_ITEM, None, None, "Image/video…")]], |
| "canva": [[('.cv-rail-btn[data-panel="uploads"]', None, '.cv-rail-btn[data-panel="uploads"].on', "the Uploads tab"), |
| (".cv-upload", None, None, "Upload files")]], |
| "photoshop": [[(".ps-menu", "File", None, "the File menu"), (PLACE_ITEM, None, None, "Place Embedded…")]], |
| "slides": [[(".sl-menu", "Insert", None, "the Insert menu"), (".pop-menu .menu-item.has-sub", "Image", None, "Image"), |
| (PLACE_ITEM, None, None, "Upload from computer")], |
| [('.sl-toolbar [data-open="images"]', None, None, "the Insert image button"), (PLACE_ITEM, None, None, "Upload from computer")]], |
| "powerpoint": [[('.pp-tabs [data-tab="insert"]', None, '.pp-tabs [data-tab="insert"].on', "the Insert tab"), |
| ('.pp-ribbon [data-open="pictures"]', None, None, "Pictures"), (PLACE_ITEM, None, None, "This Device…")]], |
| "inkscape": [[(".ik-menu", "File", None, "the File menu"), (PLACE_ITEM, None, None, "Import…")], |
| [('.ik-commands [data-open="files"]', None, None, "the Import button")]], |
| } |
| SHORTCUT = {"figma": "ctrl+shift+k", "inkscape": "ctrl+i"} |
| FILE_WINDOW = {"mac": "Finder window", "windows": "Open dialog", "linux": "file chooser"} |
|
|
| FIELD_NAME = {"x": "X", "y": "Y", "width": "W", "height": "H", "angle": "rotation", "font_size": "font size", |
| "line_height": "line height", "letter_spacing": "letter spacing", "color": "text colour", |
| "background": "background colour", "opacity": "opacity"} |
|
|
|
|
| |
| |
| |
| TEXT_ROUTES = { |
| "panel": {"font": ".inspector .font-btn", "size": '.inspector [data-field="font_size"] input', |
| "toggle": '.inspector button[title="{t}"]', "color": ("hex", '.inspector [data-field="color"] .hex-input'), |
| "align": ("segmented", '.inspector .segmented button[data-value="{v}"]')}, |
| "canva": {"font": ".cv-context .cv-font", "size": ".cv-context .cv-size input", |
| "toggle": '.cv-context button[title="{t}"]', "color": ("panel", ".cv-context .cv-textcolor"), |
| "align": ("cycle", '.cv-context button[title="Alignment"]')}, |
| "photoshop": {"font": ".ps-options .ps-font", "size": '.ps-options input[aria-label="Font size"]', |
| "toggle": '.inspector button[title="{t}"]', "color": ("popover", ".ps-options .ps-color .color-chip"), |
| "align": ("segmented", '.ps-options .segmented button[data-value="{v}"]')}, |
| |
| |
| "slides": {"font": ".sl-toolbar .sl-font", "size": ".sl-toolbar .sl-size input", |
| "toggle": '.sl-toolbar button[title="{t}"]', "color": ("slmenu", ".sl-toolbar .sl-textcolor"), |
| "align": ("slpop", ".sl-toolbar .sl-align")}, |
| |
| |
| "powerpoint": {"font": ".pp-ribbon .pp-font", "size": ".pp-ribbon .pp-size input", |
| "toggle": '.pp-ribbon button[title="{t}"]', "color": ("ppgallery", ".pp-ribbon .pp-textcolor-drop"), |
| "align": ("ppmenu", ".pp-ribbon .pp-align"), |
| "pre": ('.pp-tabs [data-tab="home"]', '.pp-tabs [data-tab="home"].on')}, |
| |
| "figma": {"font": ".inspector .font-btn", "size": '.inspector [data-field="font_size"] input', "toggle": None, |
| "color": ("hex", '.inspector [data-field="color"] .hex-input'), |
| "align": ("segmented", '.inspector .segmented button[data-value="{v}"]'), |
| "pre": ('.fg-tabs [data-tab="design"]', '.fg-tabs [data-tab="design"].on')}, |
| |
| |
| "inkscape": {"font": ".ik-controls .ik-font", "size": ".ik-controls .ik-size input", "toggle": None, |
| "color": ("ikfill", '[data-command="dlg_fill"]'), "align": ("ikmenu", ".ik-controls .ik-align"), |
| "pre": ('[data-command="toolText"]', '.ik-tool.on[data-command="toolText"]')}, |
| } |
| |
| |
| ALWAYS_NATIVE = {"canva", "figma", "inkscape", "slides", "powerpoint"} |
| POPOVER_FIELDS = { |
| "canva": {"spacing": '.cv-context button[title="Spacing"]', "opacity": '.cv-context button[title="Transparency"]'}, |
| } |
| TOOLBAR_NAME = {"canva": "the text toolbar", "photoshop": "the options bar", "slides": "the toolbar", |
| "powerpoint": "the Home ribbon", "inkscape": "the tool controls bar", "panel": "the properties panel", |
| "figma": "the Typography section"} |
|
|
|
|
| def num(v: float) -> str: |
| r = round(float(v), 2) |
| return str(int(r)) if r == int(r) else str(r) |
|
|
|
|
| def gui_expressible(pack, task) -> str | None: |
| """Why the GUI cannot reproduce this task exactly, or None if it can.""" |
| gt = pack.gt_doc(task) |
| if any(e.runs for e in gt.elements): |
| return "mixed-style text runs are not editable in the GUI" |
| if any(e.kind == "shape" for e in gt.elements): |
| return "shape elements" |
| return None |
|
|
|
|
| |
|
|
|
|
| @dataclass |
| class Style: |
| """How this episode's agent behaves. `None` rng = the plain, fixed behaviour.""" |
|
|
| rng: random.Random | None = None |
| drag_assets: float = 0.5 |
| |
| |
| drag_files: float = 0.0 |
| shortcut: float = 0.3 |
| open_by: tuple = (0.45, 0.35) |
| place_box: float = 0.5 |
| resize_window: float = 0.4 |
| upload_all: float = 0.5 |
| place_text: float = 0.7 |
| triple_click: float = 0.7 |
| tab_commit: float = 0.45 |
| one_per_turn: float = 0.4 |
| two_per_turn: float = 0.25 |
| observe: float = 0.3 |
| drag_text: float = 0.3 |
| click_away: float = 0.4 |
| |
| |
| drag_only: float = 0.5 |
| drag_refine: float = 0.35 |
| drag_rotate: float = 0.6 |
| pick_doc_font: float = 0.5 |
| native_text: float = 0.7 |
| view_reference: float = 0.25 |
| mistake: float = 0.12 |
| max_mistakes: int = 2 |
| shuffle_props: bool = True |
|
|
| def chance(self, p: float) -> bool: |
| return bool(self.rng) and self.rng.random() < p |
|
|
|
|
| class Say: |
| """Short, varied thoughts. Only used when recording.""" |
|
|
| def __init__(self, rng: random.Random | None): |
| self.rng = rng |
|
|
| def __call__(self, *options: str) -> str: |
| return self.rng.choice(options) if self.rng else options[0] |
|
|
|
|
| @dataclass |
| class Recording: |
| """What the agent did, in the rollout harness's message shape.""" |
|
|
| out_dir: Path |
| stem: str |
| image_format: str = "webp" |
| messages: list[dict] = field(default_factory=list) |
| images: list[str] = field(default_factory=list) |
| n_calls: int = 0 |
| masked: int = 0 |
|
|
| def save_image(self, data: bytes, name: str) -> str: |
| from PIL import Image |
|
|
| rel = f"images/{self.stem}/{name}.{self.image_format}" |
| path = self.out_dir / rel |
| path.parent.mkdir(parents=True, exist_ok=True) |
| if self.image_format == "png": |
| path.write_bytes(data) |
| else: |
| img = Image.open(io.BytesIO(data)).convert("RGB") |
| img.save(path, "WEBP", quality=90, method=4) if self.image_format == "webp" else \ |
| img.save(path, "JPEG", quality=90) |
| self.images.append(rel) |
| return rel |
|
|
|
|
| class GuiAgent: |
| """Rebuilds a ground-truth document through the GUI tools only.""" |
|
|
| def __init__(self, server: str, env, episode_id: str, skin: str, *, style: Style | None = None, |
| recording: Recording | None = None, verbose: bool = False, mode: str = "reference"): |
| self.server = server.rstrip("/") |
| self.mode = mode |
| self.env = env |
| self.eid = episode_id |
| self.skin = skin |
| self.style = style or Style() |
| self.say = Say(self.style.rng) |
| self.rec = recording |
| self.verbose = verbose |
| self.actions = 0 |
| self.turns = 0 |
| self.last = None |
| self.mistakes: list[dict] = [] |
| self._pending_thought = "" |
|
|
| |
|
|
| def _post(self, path, body): |
| req = urllib.request.Request(self.server + path, data=json.dumps(body).encode(), |
| headers={"content-type": "application/json"}) |
| with urllib.request.urlopen(req, timeout=30) as r: |
| return json.loads(r.read()) |
|
|
| def locate(self, selector, text=None): |
| return self._post(f"/api/debug/gui/{self.eid}/locate", {"selector": selector, "text": text}) |
|
|
| def js(self, code): |
| return self._post(f"/api/debug/gui/{self.eid}/eval", {"js": code}) |
|
|
| def _exec(self, tool, args): |
| out = self.env.step(CallToolAction(tool_name=tool, arguments=args)) |
| self.actions += 1 |
| self.last = out |
| if out.observation.error is not None: |
| raise RuntimeError(f"{tool}{args}: {out.observation.error.message}") |
| if self.verbose: |
| print(f" {tool} {args}") |
| return out |
|
|
| def screenshot(self) -> bytes: |
| out = self.env.step(CallToolAction(tool_name="screenshot", arguments={})) |
| block = (out.observation.result or {}).get("content", [{}])[0] |
| return base64.b64decode(block["data"]) |
|
|
| def turn(self, thought: str, calls: list[tuple[str, dict]], train: bool = True): |
| """Execute one turn; when recording, append it and the screenshot that follows.""" |
| if self._pending_thought: |
| thought = f"{self._pending_thought} {thought}".strip() |
| self._pending_thought = "" |
| results = [self._exec(tool, args) for tool, args in calls] |
| self.turns += 1 |
| if self.rec is None: |
| return results[-1] |
| ids = [f"call_{self.rec.n_calls + i}" for i in range(len(calls))] |
| self.rec.n_calls += len(calls) |
| msg = {"role": "assistant", "content": thought, "tool_calls": [ |
| {"id": cid, "type": "function", |
| "function": {"name": tool, "arguments": json.dumps(args, ensure_ascii=False)}} |
| for cid, (tool, args) in zip(ids, calls, strict=True)]} |
| if not train: |
| msg["train"] = False |
| self.rec.masked += 1 |
| self.rec.messages.append(msg) |
| done = results[-1].done |
| for i, (cid, (tool, _), out) in enumerate(zip(ids, calls, results, strict=True)): |
| text = _result_text(out) |
| parts = [{"type": "text", "text": text}] if text else [] |
| for k, block in enumerate((out.observation.result or {}).get("content", [])): |
| if block.get("type") == "image": |
| rel = self.rec.save_image(base64.b64decode(block["data"]), f"t{self.turns:03d}-{tool}{k}") |
| parts.append({"type": "image", "image": rel}) |
| if i == len(calls) - 1 and not done: |
| rel = self.rec.save_image(self.screenshot(), f"t{self.turns:03d}") |
| parts.append({"type": "image", "image": rel}) |
| content: list | str = parts if any(p["type"] == "image" for p in parts) else text |
| self.rec.messages.append({"role": "tool", "tool_call_id": cid, "name": tool, "content": content}) |
| return results[-1] |
|
|
| |
|
|
| def visible(self, selector, text=None, why=""): |
| """Where to click a control as it is on screen now, scrolling its panel into view first. |
| |
| "Visible" means what a person would see: not clipped by its scrolling |
| panel, and the control itself is under the pointer (not a popover or the |
| menu bar). Each scroll is its own turn, with a screenshot, as it would be |
| for a model. |
| """ |
| for _ in range(12): |
| hits = self.locate(selector, text) |
| if not hits: |
| raise RuntimeError(f"not on screen: {selector} {text or ''}") |
| r = hits[0] |
| if r.get("visible", True) and 0 <= r["x"] < VIEW_W and 0 <= r["y"] < VIEW_H: |
| return r |
| sc = r.get("scroller") |
| if not sc: |
| raise RuntimeError(f"not reachable ({'covered' if r.get('covered') else 'off screen'}): " |
| f"{selector} at {self.at(r)}") |
| what = why or "it" |
| self.turn(self.say(f"{what[0].upper() + what[1:]} is out of view; scroll the panel {sc['dir']}.", |
| f"Scroll {sc['dir']} to bring {what} into view."), |
| [("scroll", {"coordinate": [round(sc["x"]), round(sc["y"])], "scroll_direction": sc["dir"], |
| "scroll_amount": 2})]) |
| raise RuntimeError(f"could not scroll to {selector}") |
|
|
| @staticmethod |
| def at(r) -> list[int]: |
| return [round(r["x"]), round(r["y"])] |
|
|
| def click(self, selector, thought, text=None, clicks=1, why="", train=True): |
| r = self.visible(selector, text, why) |
| tool = {1: "left_click", 2: "double_click", 3: "triple_click"}[clicks] |
| return self.turn(thought, [(tool, {"coordinate": self.at(r)})], train=train) |
|
|
| def canvas_point(self, x, y): |
| """Screen position of a canvas point, or None when it is off the visible stage.""" |
| p = self.js(f"""(() => {{ const s = window.__app.stage, r = s.el.getBoundingClientRect(); |
| const [sx, sy] = s.toScreen({x}, {y}); |
| const X = r.left + sx, Y = r.top + sy; |
| return (X > r.left + 4 && X < r.right - 4 && Y > r.top + 4 && Y < r.bottom - 4) ? [X, Y] : null; }})()""") |
| return [round(p[0]), round(p[1])] if p else None |
|
|
| def open(self, panel): |
| active = ACTIVE[panel].get(self.skin) |
| if active and self.locate(active): |
| if self.skin == "powerpoint" and panel == "inspector": |
| self.pp_pane(top="shape", sub="size") |
| return |
| name = PANEL_NAME[panel][self.skin] |
| path = OPENERS[panel][self.skin] |
| steps = [(path, None)] if isinstance(path, str) else path |
| for k, (sel, done) in enumerate(steps): |
| if done and self.locate(done): |
| continue |
| last = k == len(steps) - 1 |
| where = STEP_NAME.get(sel, f"the {name} tab") |
| self.click(sel, self.say(f"Open the {name} panel.", f"Switch to the {name} panel.", |
| f"The {name} panel has what I need; open it.") if last else |
| self.say(f"Go to {where}.", f"Open {where} to find {name}."), why=where) |
| if self.skin == "powerpoint" and panel == "inspector": |
| self.pp_pane(top="shape", sub="size") |
|
|
| def ensure_for(self, selector): |
| """A control inside the properties panel needs that panel open first.""" |
| if selector and selector.lstrip().startswith(".inspector"): |
| self.open("inspector") |
|
|
| def set_field(self, name, value, kind="num", thought=None, train=True, selector=None): |
| sel = selector or {"num": f'.inspector [data-field="{name}"] input', |
| "hex": f'.inspector [data-field="{name}"] .hex-input', |
| "slider": f'.inspector [data-field="{name}"] .range-num'}[kind] |
| label = FIELD_NAME.get(name, name.replace("_", " ")) |
| self.ensure_for(sel) |
| r = self.visible(sel, why=f"the {label} field") |
| st = self.style |
| select = ([("triple_click", {"coordinate": self.at(r)})] if st.chance(st.triple_click) |
| else [("left_click", {"coordinate": self.at(r)}), ("key", {"keys": "ctrl+a"})]) |
| commit = ("key", {"keys": "tab" if st.chance(st.tab_commit) else "enter"}) |
| shown = f"#{value}" if kind == "hex" else value |
| thought = thought or self.say(f"Set {label} to {shown}.", f"{label[0].upper() + label[1:]}: {shown}.", |
| f"Type {shown} into the {label} field.") |
| |
| |
| roll = st.rng.random() if st.rng else 1.0 |
| if roll < st.one_per_turn: |
| self.turn(thought, select, train=train) |
| self.turn("", [("type", {"text": value})], train=train) |
| out = self.turn("", [commit], train=train) |
| elif roll < st.one_per_turn + st.two_per_turn: |
| self.turn(thought, select, train=train) |
| out = self.turn("", [("type", {"text": value}), commit], train=train) |
| else: |
| out = self.turn(thought, [*select, ("type", {"text": value}), commit], train=train) |
| if train and st.chance(st.observe): |
| self._pending_thought = self.say(f"{label[0].upper() + label[1:]} now reads {shown}.", |
| f"{label[0].upper() + label[1:]} is {shown}.", "Good.") |
| return out |
|
|
| def set_number(self, name, value, what, kind="num", selector=None): |
| """A field, sometimes with a realistic slip first (masked) and then the fix.""" |
| st = self.style |
| if (st.rng and len(self.mistakes) < st.max_mistakes and st.chance(st.mistake / 3) |
| and kind == "num" and len(value.replace(".", "").replace("-", "")) >= 2): |
| wrong = _typo(value, st.rng) |
| self.set_field(name, wrong, kind, train=False, selector=selector) |
| self.mistakes.append({"kind": "typo", "field": name, "typed": wrong, "meant": value}) |
| label = FIELD_NAME.get(name, name) |
| should = {"x": f"should be at x {value}", "y": f"should be at y {value}", "width": f"should be {value} wide", |
| "height": f"should be {value} tall", "font_size": f"should be {value} px"}.get(name, f"needs {value}") |
| self._pending_thought = self.say( |
| f"The {label} field shows {wrong}, but {what} {should}; fix it.", |
| f"That's a typo: {label} reads {wrong}. Correct it to {value}.") |
| self.set_field(name, value, kind, selector=selector) |
|
|
| def toggle(self, title, want: bool, template='.inspector button[title="{t}"]'): |
| sel = template.format(t=title) |
| probe = f"""(() => {{ |
| const b = [...document.querySelectorAll('{sel}')].find((e) => e.offsetParent !== null); |
| return b ? b.classList.contains('on') : null; }})()""" |
| self.ensure_for(sel) |
| state = self.js(probe) |
| if state is None and not sel.startswith(".inspector"): |
| |
| return self.toggle(title, want) |
| if state is None: |
| raise RuntimeError(f"no {title} toggle on screen") |
| if state is not None and state != want: |
| self.click(sel, |
| self.say(f"Turn {title.lower()} {'on' if want else 'off'}.", |
| f"{title} should be {'on' if want else 'off'}."), why=f"the {title} button") |
|
|
| def deselect(self, thought=None): |
| st = self.style |
| if not st.rng: |
| return self.turn(thought or "Deselect.", [("key", {"keys": "escape"}), ("key", {"keys": "escape"})]) |
| spot = self.empty_canvas() if st.chance(st.click_away) else None |
| if spot: |
| return self.turn(thought or self.say("Click empty canvas to deselect.", "Deselect by clicking off the design."), |
| [("left_click", {"coordinate": spot})]) |
| self.turn(thought or self.say("Deselect before the next layer.", "Clear the selection.", |
| "Press escape to deselect."), [("key", {"keys": "escape"})]) |
| if self.js("window.__app.store.selection.length"): |
| self.turn("", [("key", {"keys": "escape"})]) |
|
|
| def empty_canvas(self): |
| """A point on the grey stage around the artboard that hits nothing (or None).""" |
| p = self.js("""(() => { |
| const st = window.__app.stage.el, r = st.getBoundingClientRect(); |
| const ab = document.querySelector('.artboard').getBoundingClientRect(); |
| const pts = [[r.left + 24, r.top + 24], [r.right - 24, r.top + 24], [r.left + 24, (r.top + r.bottom) / 2], |
| [r.right - 24, (r.top + r.bottom) / 2], [(ab.left + ab.right) / 2, ab.top - 14]]; |
| for (const [x, y] of pts) { |
| if (x > ab.left - 4 && x < ab.right + 4 && y > ab.top - 4 && y < ab.bottom + 4) continue; |
| const at = document.elementFromPoint(x, y); |
| if (at && (at === st || at.classList.contains('stage-inner') || at.classList.contains('stage'))) return [x, y]; |
| } |
| return null; })()""") |
| return [round(p[0]), round(p[1])] if p else None |
|
|
| |
|
|
| def _describe(self, el, assets) -> str: |
| if el.kind == "text": |
| line = el.text.split("\n")[0] |
| return f"the text “{line[:40]}{'…' if len(line) > 40 else ''}”" |
| kind = assets.get(el.asset_id, {}).get("kind", "image") |
| return {"photo": f"the photo {el.asset_id}", "background": f"the background image {el.asset_id}", |
| "graphic": f"the graphic {el.asset_id}"}.get(kind, f"asset {el.asset_id}") |
|
|
| def add_image(self, el, assets): |
| what = self._describe(el, assets) |
| images = list(assets) |
| if self.skin == "canva" and self.js(f"window.__app.uploaded?.has({json.dumps(el.asset_id)})"): |
| self.from_uploads(el, what, assets) |
| else: |
| self.from_files(el, what, images) |
| if not self.place(el, what, ("width", "height", "x", "y")): |
| self.open("inspector") |
| self.set_number("width", num(el.width), what) |
| self.set_number("height", num(el.height), what) |
| for name in self._order(["x", "y"]): |
| self.set_number(name, num(getattr(el, name)), what) |
| self.finish_common(el, what) |
|
|
| |
|
|
| def chooser_os(self): |
| return self.js("document.querySelector('.fc')?.dataset.os || null") |
|
|
| def open_file_window(self, what): |
| """Open the computer's file window the way this layout does it.""" |
| st = self.style |
| if self.chooser_os(): |
| return |
| if self.skin in SHORTCUT and st.chance(st.shortcut): |
| keys = SHORTCUT[self.skin] |
| self.turn(self.say(f"Open the file window with {keys} to add {what}.", f"{keys} opens the file picker; I need {what}."), |
| [("key", {"keys": keys})]) |
| else: |
| route = st.rng.choice(FILE_ROUTES[self.skin]) if st.rng else FILE_ROUTES[self.skin][0] |
| for k, (sel, text, done, name) in enumerate(route): |
| if done and self.locate(done): |
| continue |
| first, last = k == 0, k == len(route) - 1 |
| thought = (self.say(f"To add {what} I need its file: open {name}.", f"Images come from files here; open {name}.") |
| if first else self.say(f"Choose {name}.", f"Click {name}.", f"{name} opens the file window.") if last |
| else self.say(f"Then {name}.", f"Open {name}.")) |
| self.click(sel, thought, text=text, why=name) |
| if not self.chooser_os(): |
| raise RuntimeError(f"{self.skin}: the file window did not open") |
|
|
| def file_item(self, asset_id): |
| return f'.fc .fc-item[data-file="{asset_id}"]' |
|
|
| def from_files(self, el, what, images): |
| st = self.style |
| self.open_file_window(what) |
| win = FILE_WINDOW.get(self.chooser_os(), "file window") |
| fname = self.js(f"document.querySelector('{self.file_item(el.asset_id)}')?.title") |
| if fname is None: |
| raise RuntimeError(f"{el.asset_id} is not in the file window") |
| canva = self.skin == "canva" |
| pending = [a for a in images if not self.js(f"window.__app.uploaded?.has({json.dumps(a)})")] if canva else [] |
| if canva and len(pending) > 1 and st.chance(st.upload_all): |
| return self.upload_all(el, what, pending, win) |
| target = self.canvas_point(el.x + el.width / 2, el.y + el.height / 2) |
| if target and st.chance(st.drag_files) and self.uncover(target, what, win, el.asset_id): |
| self.maybe_wrong_file(el, images, what) |
| item = self.visible(self.file_item(el.asset_id), why=fname) |
| self.turn(self.say(f"Drag {fname} out of the {win} onto the canvas where {what} sits.", |
| f"Drop {fname} straight onto the canvas, roughly in place."), |
| [("left_click_drag", {"start_coordinate": self.at(item), "coordinate": target})]) |
| self.confirm_import() |
| self.close_file_window(win) |
| return |
| self.maybe_wrong_file(el, images, what) |
| self.open_file(el.asset_id, fname, what, win) |
| self.confirm_import() |
| if self.skin == "figma": |
| self.click_to_place(el, what) |
| elif canva: |
| self.from_uploads(el, what, None, fresh=True) |
|
|
| def maybe_wrong_file(self, el, images, what): |
| """Sometimes click the wrong file first (masked), then the right one.""" |
| st = self.style |
| if not (st.rng and len(self.mistakes) < st.max_mistakes and len(images) > 1 and st.chance(st.mistake / 2)): |
| return |
| wrong = st.rng.choice([a for a in images if a != el.asset_id]) |
| name = self.js(f"document.querySelector('{self.file_item(wrong)}')?.title") |
| if not name: |
| return |
| self.click(self.file_item(wrong), self.say(f"Select the file for {what}.", "Pick the image file."), why=name, train=False) |
| self.mistakes.append({"kind": "wrong_file", "selected": wrong, "meant": el.asset_id}) |
| self._pending_thought = self.say(f"That selected {name}; {what} is {el.asset_id}.", |
| f"Wrong file: {name} is highlighted, but I need {el.asset_id}.") |
|
|
| def open_file(self, asset_id, fname, what, win): |
| st = self.style |
| item = self.file_item(asset_id) |
| roll = st.rng.random() if st.rng else 0.0 |
| if roll < st.open_by[0]: |
| self.click(item, self.say(f"Select {fname}.", f"Click {fname} in the {win}."), why=fname) |
| label = self.js("document.querySelector('.fc [data-fc=\"open\"]').textContent") |
| self.click('.fc [data-fc="open"]', self.say("Open it.", f"Click {label}."), why="the Open button") |
| elif roll < st.open_by[0] + st.open_by[1]: |
| self.click(item, self.say(f"Double-click {fname} to open it.", f"Open {fname} with a double-click."), clicks=2, why=fname) |
| else: |
| self.click(item, self.say(f"Select {fname}.", f"Click {fname}."), why=fname) |
| self.turn(self.say("Press enter to open it.", "Enter opens the selected file."), [("key", {"keys": "enter"})]) |
| if self.chooser_os(): |
| raise RuntimeError(f"{fname} did not open") |
|
|
| def upload_all(self, el, what, pending, win): |
| """Template: select every image in the window and upload them together.""" |
| st = self.style |
| if st.chance(0.5): |
| first = self.visible(self.file_item(pending[0]), why="the first file") |
| self.turn(self.say("Upload all the images at once: select each file, then Open.", "Select every image file and upload them together."), |
| [("left_click", {"coordinate": self.at(first)})]) |
| for a in pending[1:]: |
| r = self.visible(self.file_item(a), why=a) |
| self.turn("", [("left_click", {"coordinate": self.at(r), "text": "ctrl"})]) |
| else: |
| first = self.visible(self.file_item(pending[0]), why="the first file") |
| self.turn(self.say("Select all the image files with ctrl+a and upload them together.", "Upload everything at once: select all."), |
| [("left_click", {"coordinate": self.at(first)}), ("key", {"keys": "ctrl+a"})]) |
| self.click('.fc [data-fc="open"]', self.say("Upload them.", "Open them to upload."), why="the Open button") |
| self.from_uploads(el, what, None, fresh=True) |
|
|
| def uncover(self, target, what, win, asset_id) -> bool: |
| """Make sure the file window is not over the drop spot: shrink it or move it aside.""" |
| def geo(): |
| return self.js("""(() => { const r = document.querySelector('.fc').getBoundingClientRect(); |
| return [r.left, r.top, r.right, r.bottom]; })()""") |
| tx, ty, m = target[0], target[1], 36 |
| x0, t, r, b = geo() |
| if not (x0 - m < tx < r + m and t - m < ty < b + m): |
| return True |
| st = self.style |
| MINW, MINH = 460, 300 |
| edges = [] |
| if tx - m - x0 >= MINW: |
| edges.append(("e", [tx - m, (t + b) / 2], "right")) |
| if r - (tx + m) >= MINW: |
| edges.append(("w", [tx + m, (t + b) / 2], "left")) |
| if ty - m - t >= MINH: |
| edges.append(("s", [(x0 + r) / 2, ty - m], "bottom")) |
| if edges and st.chance(st.resize_window): |
| handle, to, side = st.rng.choice(edges) |
| h = self.locate(f".fc .fc-rz-{handle}")[0] |
| start = self.at(h) |
| end = [round(to[0]), start[1]] if handle in "ew" else [start[0], round(to[1])] |
| self.turn(self.say(f"The {win} covers where {what} goes; drag its {side} edge in to make it smaller.", |
| f"Shrink the {win} from its {side} edge so the drop spot is clear."), |
| [("left_click_drag", {"start_coordinate": start, "coordinate": end})]) |
| else: |
| grab = self.js("""(() => { |
| for (const bar of document.querySelectorAll('.fc .fc-drag')) { |
| const r = bar.getBoundingClientRect(), y = r.top + Math.min(r.height / 2, 18); |
| for (let x = r.left + r.width * 0.35; x < r.right - 8; x += 12) { |
| const at = document.elementFromPoint(x, y); |
| if (at && bar.contains(at) && !at.closest('button, input, select')) return [x, y]; |
| } |
| } |
| return null; })()""") |
| if not grab: |
| return False |
| item = self.locate(self.file_item(asset_id)) |
| ix, iy = (item[0]["x"], item[0]["y"]) if item else (x0 + 260, t + 140) |
| |
| moves = [(tx + m - x0, 0), (tx - m - r, 0), (0, ty + m - t), (0, ty - m - b)] |
| ok = [(dx, dy) for dx, dy in moves |
| if 8 < grab[0] + dx < VIEW_W - 8 and 8 < grab[1] + dy < VIEW_H - 8 and 30 < ix + dx < VIEW_W - 30 |
| and 30 < iy + dy < VIEW_H - 30 and t + dy >= 0] |
| if not ok: |
| return False |
| dx, dy = min(ok, key=lambda d: abs(d[0]) + abs(d[1])) |
| self.turn(self.say(f"The {win} is over the spot where {what} goes; drag it aside by its title bar.", |
| f"Move the {win} out of the way first."), |
| [("left_click_drag", {"start_coordinate": [round(grab[0]), round(grab[1])], |
| "coordinate": [round(grab[0] + dx), round(grab[1] + dy)]})]) |
| x0, t, r, b = geo() |
| return not (x0 - 4 < tx < r + 4 and t - 4 < ty < b + 4) |
|
|
| def confirm_import(self): |
| """Vector asks how to import a bitmap; the defaults (embed) are right.""" |
| if self.skin != "inkscape": |
| return |
| self.click(".ik-import .btn-primary", self.say("Keep Embed and click OK.", "The import options are fine; OK."), |
| why="the OK button") |
|
|
| def close_file_window(self, win): |
| if not self.chooser_os(): |
| return |
| st = self.style |
| roll = st.rng.random() if st.rng else 0.0 |
| if roll < 0.3: |
| self.turn(self.say(f"Done with the {win}; press escape.", f"Close the {win} with escape."), |
| [("key", {"keys": "escape"})]) |
| if self.chooser_os(): |
| |
| ctrls = ['.fc [data-fc="close"]', '.fc [data-fc="cancel"]'] if roll > 0.65 else ['.fc [data-fc="cancel"]', '.fc [data-fc="close"]'] |
| seen = [c for c in ctrls if any(r.get("visible", True) and 0 <= r["x"] < VIEW_W and 0 <= r["y"] < VIEW_H |
| for r in self.locate(c))] |
| if seen: |
| self.click(seen[0], self.say(f"Close the {win}.", f"The {win} is no longer needed; close it."), why="the close button") |
| else: |
| self.turn(self.say(f"The {win}'s buttons are off screen; press escape to close it.", f"Escape closes the {win}."), |
| [("key", {"keys": "escape"})]) |
|
|
| def click_to_place(self, el, what): |
| """Studio: the opened image follows the pointer until a click (or a dragged box) places it.""" |
| st = self.style |
| if not self.locate(".stage.placing"): |
| return |
| a = self.canvas_point(el.x, el.y) |
| z = self.canvas_point(el.x + el.width, el.y + el.height) |
| c = self.canvas_point(el.x + el.width / 2, el.y + el.height / 2) or self.canvas_point( |
| self.js("window.__app.store.doc.width") / 2, self.js("window.__app.store.doc.height") / 2) |
| if a and z and z[0] - a[0] > 12 and st.chance(st.place_box): |
| self.turn(self.say(f"Drag out {what}'s box from its top-left corner.", f"Draw the image's box where {what} sits."), |
| [("left_click_drag", {"start_coordinate": a, "coordinate": z})]) |
| else: |
| self.turn(self.say(f"Click to place it where {what} goes.", f"Click on the canvas to drop {what} there."), |
| [("left_click", {"coordinate": c})]) |
|
|
| def from_uploads(self, el, what, assets, fresh=False): |
| """Template: the file is in Uploads; click it in or drag it into place.""" |
| st = self.style |
| self.open("assets") |
| if not fresh and assets and st.rng and len(self.mistakes) < st.max_mistakes and st.chance(st.mistake / 2): |
| others = [a for a in assets if a != el.asset_id and self.js(f"window.__app.uploaded?.has({json.dumps(a)})")] |
| if others: |
| wrong = st.rng.choice(others) |
| self.click(f'.asset-tile[data-asset="{wrong}"]', self.say(f"Add {what}.", f"Place {what}."), |
| why=f"asset {wrong}", train=False) |
| self.mistakes.append({"kind": "wrong_asset", "added": wrong, "meant": el.asset_id}) |
| self.turn(self.say(f"That added {wrong}, not {el.asset_id}. Undo it.", |
| f"Wrong image: this is {wrong}. Undo and pick {el.asset_id}."), |
| [("key", {"keys": "ctrl+z"})]) |
| tile = self.visible(f'.asset-tile[data-asset="{el.asset_id}"]', why=f"the upload {el.asset_id}") |
| target = self.canvas_point(el.x + el.width / 2, el.y + el.height / 2) if st.chance(st.drag_assets) else None |
| lead = "It's in Uploads now. " if fresh else "" |
| if target: |
| self.turn(lead + self.say(f"Drag {what} onto the canvas where it sits in the reference.", |
| f"Drop {what} roughly into place; I'll fine-tune it after."), |
| [("left_click_drag", {"start_coordinate": self.at(tile), "coordinate": target})]) |
| else: |
| self.turn(lead + self.say(f"Add {what}.", f"Click {el.asset_id} in Uploads to add {what} to the page."), |
| [("left_click", {"coordinate": self.at(tile)})]) |
|
|
| def add_text(self, el, assets, doc_fonts): |
| st, what = self.style, self._describe(el, assets) |
| if self.skin == "canva": |
| if not self.locate('.cv-rail-btn[data-panel="text"].on'): |
| self.click('.cv-rail-btn[data-panel="text"]', self.say("Open the Text panel.", "Go to Text."), |
| why="the Text tab") |
| self.click(".text-presets .btn-primary", self.say("Add a text box.", "Start a new text box."), |
| why="the add-text button") |
| |
| self.turn(self.say(f"Type {what}.", f"Replace the placeholder with {what}."), |
| [("type", {"text": el.text}), ("key", {"keys": "escape"})]) |
| else: |
| if self.skin == "powerpoint" and not self.locate('.pp-tabs [data-tab="insert"].on'): |
| self.click('.pp-tabs [data-tab="insert"]', self.say("Text Box is on the Insert tab.", "Go to Insert."), why="the Insert tab") |
| self.click('[data-command="toolText"]', self.say(f"Pick the text tool for {what}.", |
| f"Text tool, to add {what}."), why="the text tool") |
| if self.skin != "canva": |
| spot = (self.canvas_point(el.x + min(el.width, 40) / 2, el.y + el.height / 2) |
| if st.chance(st.place_text) else None) |
| if spot is None: |
| stage = self.locate(".stage")[0] |
| spot = self.at(stage) |
| self.turn(self.say(f"Click on the canvas and type {what}.", f"Place the text box and type {what}."), |
| [("left_click", {"coordinate": spot}), ("type", {"text": el.text}), ("key", {"keys": "escape"})]) |
| if st.chance(st.drag_text): |
| cur = self.js("(() => { const e = window.__app.store.selected()[0]; return e && [e.x + e.width / 2, e.y + e.height / 2]; })()") |
| src = self.canvas_point(*cur) if cur else None |
| dst = self.canvas_point(el.x + el.width / 2, el.y + el.height / 2) |
| if src and dst and abs(src[0] - dst[0]) + abs(src[1] - dst[1]) > 12: |
| self.turn(self.say(f"Drag {what} roughly to where it sits in the reference.", |
| f"Move {what} into position; exact numbers next."), |
| [("left_click_drag", {"start_coordinate": src, "coordinate": dst})]) |
| native = self.skin in TEXT_ROUTES and (self.skin in ALWAYS_NATIVE or st.chance(st.native_text)) |
| route = TEXT_ROUTES[self.skin if native else "panel"] |
| if not native: |
| self.open("inspector") |
| elif route.get("pre"): |
| sel, done = route["pre"] |
| if not self.locate(done): |
| self.click(sel, self.say(f"Go to {STEP_NAME.get(sel, 'the text controls')} for the text controls.", |
| "Switch to the text controls."), why=STEP_NAME.get(sel, "the tab")) |
| |
| |
| self.set_font(el.font_family, doc_fonts, route["font"]) |
| style_steps = [lambda: self.set_number("font_size", num(el.font_size), what, selector=route["size"])] |
| style_steps.append(lambda: self.set_text_color(el.color or "#000000", route)) |
| if el.align != "left": |
| style_steps.append(lambda: self.set_align(el.align, route)) |
| if self.skin == "figma": |
| style_steps.append(lambda: self.style_figma(el)) |
| elif self.skin == "inkscape": |
| style_steps.append(lambda: self.style_inkscape(el)) |
| elif self.skin == "slides": |
| style_steps.append(lambda: self.style_slides(el, route)) |
| elif self.skin == "powerpoint": |
| style_steps.append(lambda: self.style_powerpoint(el, route)) |
| else: |
| style_steps.append(lambda: [self.toggle(t, want, route["toggle"]) for t, want in ( |
| ("Bold", el.bold), ("Italic", el.italic), ("Underline", el.underline), ("Uppercase", el.uppercase))]) |
| for step in self._order(style_steps): |
| step() |
| pop = POPOVER_FIELDS.get(self.skin, {}) |
| if pop.get("spacing") and (el.line_height != 1 or el.letter_spacing): |
| self.click(pop["spacing"], self.say("Open Spacing for line and letter spacing.", "Spacing next."), why="the Spacing button") |
| if el.line_height != 1: |
| self.set_field("line_height", num(el.line_height), selector='.popover [data-field="line_height"] input') |
| if el.letter_spacing: |
| self.set_field("letter_spacing", num(el.letter_spacing / el.font_size * 1000), |
| selector='.popover [data-field="letter_spacing"] input') |
| if self.skin == "inkscape": |
| if el.line_height != 1: |
| self.set_field("line_height", num(el.line_height), selector='.ik-controls [data-field="line_height"] input') |
| if el.letter_spacing: |
| self.set_field("letter_spacing", num(el.letter_spacing), selector='.ik-controls [data-field="letter_spacing"] input') |
| pop = {"spacing": True} |
| if self.skin == "slides": |
| self.spacing_slides(el) |
| pop = {"spacing": True} |
| if self.skin == "powerpoint": |
| self.spacing_powerpoint(el) |
| pop = {"spacing": True} |
| |
| self.open("inspector") |
| if not pop.get("spacing"): |
| fg = self.skin == "figma" |
| if el.line_height != 1: |
| self.set_field("line_height", f"{num(el.line_height * 100)}%" if fg else num(el.line_height)) |
| if el.letter_spacing: |
| self.set_field("letter_spacing", f"{num(el.letter_spacing)}px" if fg else num(el.letter_spacing)) |
| if not self.place(el, what, ("width", "x", "y")): |
| self.set_number("width", num(el.width), what) |
| for name in self._order(["x", "y"]): |
| self.set_number(name, num(getattr(el, name)), what) |
| self.finish_common(el, what) |
|
|
| def style_inkscape(self, el): |
| """Inkscape: the style dropdown for bold / italic; Text and Font > Features for capitals and underline.""" |
| cur = self.js("(() => { const e = window.__app.store.selected()[0]; return e && [!!e.bold, !!e.italic, !!e.underline, !!e.uppercase]; })()") |
| if cur is None: |
| return |
| want = [bool(el.bold), bool(el.italic)] |
| if cur[:2] != want: |
| name = {(False, False): "Regular", (False, True): "Italic", (True, False): "Bold", (True, True): "Bold Italic"}[tuple(want)] |
| self.click(".ik-controls .ik-style", self.say(f"Open the font style list for {name}.", f"Style: {name}."), why="the font style list") |
| self.click(".pop-menu .menu-item", self.say(f"Choose {name}.", f"{name}."), text=name, why=name) |
| if [cur[2], cur[3]] != [bool(el.underline), bool(el.uppercase)]: |
| if not self.locate('.ik-docktab.on[data-tab="text"]'): |
| self.click('[data-command="dlg_text"]', self.say("Open Text and Font for capitals and underline.", |
| "Text and Font (shift+ctrl+t) has the text features."), why="Text and Font") |
| self.click('.ik-tf .ik-subtab[data-tab="features"]', self.say("Go to the Features tab.", "Features."), why="the Features tab") |
| if cur[3] != bool(el.uppercase): |
| self.click(f'.ik-tf [data-field="uppercase"][data-value="{"true" if el.uppercase else "false"}"]', |
| self.say("Capitals: all caps." if el.uppercase else "Capitals: normal.", "Set the capitals."), why="Capitals") |
| if cur[2] != bool(el.underline): |
| self.click(f'.ik-tf [data-field="underline"][data-value="{"true" if el.underline else "false"}"]', |
| self.say("Decoration: underline." if el.underline else "Decoration: none.", "Set the decoration."), why="Decoration") |
| self.click(".ik-tf .ik-apply", self.say("Apply.", "Click Apply to use it."), why="the Apply button") |
|
|
| def style_slides(self, el, route): |
| """Slides: B / I / U in the toolbar; capitals in Format > Text > Capitalization.""" |
| for t, want in (("Bold", el.bold), ("Italic", el.italic), ("Underline", el.underline)): |
| self.toggle(t, want, route["toggle"]) |
| cur = self.js("!!window.__app.store.selected()[0]?.uppercase") |
| if cur != bool(el.uppercase): |
| name = "UPPERCASE" if el.uppercase else "As typed" |
| self.click(".sl-menu", self.say("Capitals are under Format > Text > Capitalization; open Format.", "Format menu for the capitals."), |
| text="Format", why="the Format menu") |
| self.click(".pop-menu .menu-item.has-sub", self.say("Text.", "Go to Text."), text="Text", why="Text") |
| self.click(".menu-sub .menu-item.has-sub", self.say("Capitalization.", "Then Capitalization."), text="Capitalization", |
| why="Capitalization") |
| self.click(".menu-sub .menu-item", self.say(f"{name}.", f"Choose {name}."), text=name, why=name) |
|
|
| def spacing_slides(self, el): |
| """Slides: Line & paragraph spacing; a preset when it is one, else Custom spacing (line and letter spacing).""" |
| presets = {1.0: "Single", 1.15: "1.15", 1.5: "1.5", 2.0: "Double"} |
| preset = presets.get(round(el.line_height, 4)) |
| if el.line_height == 1 and not el.letter_spacing: |
| return |
| self.click(".sl-toolbar .sl-spacing", self.say("Open Line & paragraph spacing.", "Spacing menu next."), |
| why="the line spacing button") |
| if preset and not el.letter_spacing: |
| self.click(".pop-menu .menu-item", self.say(f"{preset} spacing.", f"Choose {preset}."), text=preset, why=preset) |
| return |
| self.click(".pop-menu .menu-item", self.say("Custom spacing.", "Open Custom spacing."), text="Custom spacing", |
| why="Custom spacing") |
| if el.line_height != 1: |
| self.set_field("line_height", num(el.line_height), selector='.sl-spacing-dialog [data-field="line_height"] input') |
| if el.letter_spacing: |
| self.set_field("letter_spacing", num(el.letter_spacing), selector='.sl-spacing-dialog [data-field="letter_spacing"] input') |
| self.click(".sl-spacing-dialog .modal-foot .btn-primary", self.say("Apply.", "Click Apply."), why="Apply") |
|
|
| def slides_color(self, hexv): |
| """In an open Slides colour menu: the swatch when the palette has it, else CUSTOM + and the hex.""" |
| sw = f'.sl-pop .slc-sw[data-color="#{hexv.lower()}"]' |
| if self.locate(sw) and self.style.chance(0.8): |
| self.click(sw, self.say(f"#{hexv} is in the palette; click it.", f"Pick the #{hexv} swatch."), why="the swatch") |
| return |
| self.click(".sl-pop .slc-add", self.say("Click + under CUSTOM for a custom colour.", "CUSTOM: the + button."), |
| why="the add custom colour button") |
| self.turn(self.say(f"The Hex field is selected; type #{hexv} and press enter for OK.", f"Hex: #{hexv}, then OK."), |
| [("type", {"text": f"#{hexv}"}), ("key", {"keys": "enter"})]) |
|
|
| def style_powerpoint(self, el, route): |
| """PowerPoint: B / I / U on the Home tab; All Caps in the Font dialog (More Font Options > Font...).""" |
| for t, want in (("Bold", el.bold), ("Italic", el.italic), ("Underline", el.underline)): |
| self.toggle(t, want, route["toggle"]) |
| if self.js("!!window.__app.store.selected()[0]?.uppercase") != bool(el.uppercase): |
| self.pp_font_dialog("Font...") |
| self.click('.pp-fontdlg [data-field="uppercase"]', self.say("Tick All Caps." if el.uppercase else "Untick All Caps.", "All Caps."), |
| why="All Caps") |
| self.click(".pp-fontdlg .modal-foot .btn-primary", self.say("OK.", "Click OK."), why="OK") |
|
|
| def pp_font_dialog(self, item): |
| self.click(".pp-ribbon .pp-morefont", self.say("Open More Font Options.", "The ... after the font colour."), why="More Font Options") |
| if item != "Font...": |
| self.click(".pop-menu .menu-item.has-sub", self.say("Character Spacing.", "Go to Character Spacing."), text="Character Spacing", |
| why="Character Spacing") |
| self.click(".menu-sub .menu-item", self.say(f"{item}", f"Choose {item}"), text=item, why=item) |
| else: |
| self.click(".pop-menu .menu-item", self.say("Font... opens the Font dialog.", "Open the Font dialog."), text="Font...", why="Font...") |
|
|
| def pp_dropdown(self, sel, label, why): |
| self.click(sel, self.say(f"Open {why}.", f"{why[0].upper() + why[1:]} dropdown."), why=why) |
| self.click(".pp-ddmenu .menu-item", self.say(f"{label}.", f"Pick {label}."), text=label, why=label) |
|
|
| def spacing_powerpoint(self, el): |
| """PowerPoint: Line Spacing (a preset, or Line Spacing Options > Multiple, At); Character Spacing > More Spacing.""" |
| lh = round(el.line_height, 4) |
| if lh != 1: |
| self.click(".pp-ribbon .pp-linespacing", self.say("Open Line Spacing.", "Line spacing next."), why="Line Spacing") |
| if lh in (1.5, 2.0, 2.5, 3.0) and self.style.chance(0.8): |
| self.click(".pop-menu .menu-item", self.say(f"{lh:.1f}.", f"Choose {lh:.1f}."), text=f"{lh:.1f}", why=f"{lh:.1f}") |
| else: |
| self.click(".pop-menu .menu-item", self.say("Line Spacing Options...", "Open the Paragraph dialog."), text="Line Spacing Options...", |
| why="Line Spacing Options") |
| self.pp_dropdown(".pp-paragraph .pp-para-spacing", "Multiple", "the Line Spacing list") |
| self.set_field("line_height", num(el.line_height), selector='.pp-paragraph [data-field="line_height"] input') |
| self.click(".pp-paragraph .modal-foot .btn-primary", self.say("OK.", "Click OK."), why="OK") |
| if el.letter_spacing: |
| self.pp_font_dialog("More Spacing...") |
| self.pp_dropdown(".pp-fontdlg .pp-fd-spacing", "Expanded" if el.letter_spacing > 0 else "Condensed", "the Spacing list") |
| self.set_field("letter_spacing", num(abs(el.letter_spacing)), selector='.pp-fontdlg [data-field="letter_spacing"] input') |
| self.click(".pp-fontdlg .modal-foot .btn-primary", self.say("OK.", "Click OK."), why="OK") |
|
|
| def pp_color(self, hexv): |
| """In an open PowerPoint colour gallery: the swatch when it is there, else More Colors... and the Hex field.""" |
| sw = f'.pp-pop .ppc-sw[data-color="#{hexv.lower()}"]' |
| if self.locate(sw) and self.style.chance(0.8): |
| self.click(sw, self.say(f"#{hexv} is in the gallery; click it.", f"Pick #{hexv}."), why="the swatch") |
| return |
| self.click(".pp-pop .pp-morecolors", self.say("More Colors... for a custom colour.", "Open More Colors."), why="More Colors") |
| self.turn(self.say(f"The Hex field is selected; type {hexv.upper()} and press enter for OK.", f"Hex: {hexv.upper()}, then OK."), |
| [("type", {"text": hexv.upper()}), ("key", {"keys": "enter"})]) |
|
|
| def pp_pane(self, top=None, sub=None): |
| """Format pane tabs: Shape Options / Text Options, then the icon tab (Size & Properties, Picture, ...).""" |
| if top and self.locate(".pp-format .pp-ftop") and not self.locate(f'.pp-ftop-tab.on[data-top="{top}"]'): |
| name = {"shape": "Shape Options", "text": "Text Options"}[top] |
| self.click(f'.pp-ftop-tab[data-top="{top}"]', self.say(f"{name}.", f"Switch to {name}."), why=name) |
| if sub and self.locate(f'.pp-ftab[data-sub="{sub}"]') and not self.locate(f'.pp-ftab.on[data-sub="{sub}"]'): |
| name = {"size": "Size & Properties", "picture": "Picture", "fill": "Fill & Line"}[sub] |
| self.click(f'.pp-ftab[data-sub="{sub}"]', self.say(f"The {name} tab.", f"Go to {name}."), why=name) |
|
|
| def style_figma(self, el): |
| """Figma: Regular / Italic / Bold / Bold Italic from the weight dropdown; underline and case in Type settings.""" |
| cur = self.js("(() => { const e = window.__app.store.selected()[0]; return e && [!!e.bold, !!e.italic, !!e.underline, !!e.uppercase]; })()") |
| if cur is None: |
| return |
| want = [bool(el.bold), bool(el.italic)] |
| if cur[:2] != want: |
| name = {(False, False): "Regular", (False, True): "Italic", (True, False): "Bold", (True, True): "Bold Italic"}[tuple(want)] |
| self.click(".inspector .fg-weight", self.say(f"Open the style dropdown to pick {name}.", f"It is set in {name}; open the weight menu."), |
| why="the font style dropdown") |
| self.click(".pop-menu .menu-item", self.say(f"Choose {name}.", f"{name}."), text=name, why=name) |
| if [cur[2], cur[3]] != [bool(el.underline), bool(el.uppercase)]: |
| self.click(".inspector .fg-typeset-btn", self.say("Open Type settings for decoration and case.", "Type settings next."), |
| why="the Type settings button") |
| if cur[2] != bool(el.underline): |
| v = "underline" if el.underline else "none" |
| self.click(f'.fg-typeset [data-field="decoration"] button[data-value="{v}"]', |
| self.say(f"Decoration: {'underline' if el.underline else 'none'}.", "Set the decoration."), why="Decoration") |
| if cur[3] != bool(el.uppercase): |
| v = "upper" if el.uppercase else "none" |
| self.click(f'.fg-typeset [data-field="case"] button[data-value="{v}"]', |
| self.say(f"Case: {'uppercase' if el.uppercase else 'as typed'}.", "Set the case."), why="Case") |
| self.turn(self.say("Close Type settings.", "Done there; escape."), [("key", {"keys": "escape"})]) |
|
|
| def set_text_color(self, color, route): |
| kind, sel = route["color"] |
| hexv = color.lstrip("#")[:6] |
| if kind == "hex": |
| return self.set_field("color", hexv, kind="hex", selector=sel) |
| if kind == "ikfill": |
| if not self.locate('.ik-docktab.on[data-tab="fill"]'): |
| self.click(sel, self.say(f"Open Fill and Stroke to set the fill to #{hexv}.", "Fill and Stroke (shift+ctrl+f) for the colour."), |
| why="Fill and Stroke") |
| if not self.locate('.ik-fs .ik-subtab.on[data-tab="fill"]'): |
| self.click('.ik-fs .ik-subtab[data-tab="fill"]', self.say("Go to the Fill tab.", "Fill."), why="the Fill tab") |
| r = self.visible(".ik-fs .ik-rgba", why="the RGBA entry") |
| self.turn(self.say(f"Type {hexv}ff into RGBA and press enter.", f"RGBA: {hexv}ff."), |
| [("triple_click", {"coordinate": self.at(r)}), ("type", {"text": f"{hexv}ff"}), ("key", {"keys": "enter"})]) |
| return |
| if kind == "ppgallery": |
| self.click(sel, self.say(f"Open the Font Color gallery for #{hexv}.", f"Font colour: #{hexv}."), why="the Font Color arrow") |
| return self.pp_color(hexv) |
| if kind == "slmenu": |
| self.click(sel, self.say(f"Open the text colour menu to set #{hexv}.", f"Text colour: #{hexv}."), why="the text colour button") |
| return self.slides_color(hexv) |
| if kind == "panel": |
| if not self.locate('.cv-editpanel[data-panel="color"]'): |
| self.click(sel, self.say(f"Open Text color to set #{hexv}.", f"Colour next: #{hexv}; open the text colour panel."), |
| why="the text colour button") |
| st = self.style |
| if st.chance(0.5): |
| r = self.visible(".cv-colorpanel .cv-colsearch", why="the colour search") |
| self.turn(self.say(f"Type #{hexv} into the colour search and press enter.", f"Search #{hexv}."), |
| [("left_click", {"coordinate": self.at(r)}), ("type", {"text": f"#{hexv}"}), ("key", {"keys": "enter"})]) |
| else: |
| self.click(".cv-colorpanel .cv-addcolor", self.say("Add a new colour with the rainbow +.", "Click + for a custom colour."), |
| why="the add colour button") |
| self.turn(self.say(f"The hex field is selected; type #{hexv}.", f"Enter #{hexv}."), |
| [("type", {"text": f"#{hexv}"}), ("key", {"keys": "enter"}), ("key", {"keys": "escape"})]) |
| return |
| self.click(sel, self.say(f"Open the text colour picker to set #{hexv}.", f"Colour: open the picker for #{hexv}."), |
| why="the text colour button") |
| r = self.visible(".popover .cp-hex", why="the hex field") |
| |
| close = [] if self.skin == "photoshop" else [("key", {"keys": "escape"})] |
| self.turn(self.say(f"Enter #{hexv} as the hex value." + (" Enter is OK." if not close else ""), f"Type #{hexv} into Hex."), |
| [("triple_click", {"coordinate": self.at(r)}), ("type", {"text": f"#{hexv}"}), |
| ("key", {"keys": "enter"}), *close]) |
|
|
| def set_align(self, align, route): |
| kind, sel = route["align"] |
| if kind == "ikmenu": |
| label = {"left": "Align left", "center": "Align center", "right": "Align right"}[align] |
| self.click(sel, self.say(f"Open the alignment list to {label.lower()}.", "Alignment dropdown."), why="the alignment dropdown") |
| self.click(".pop-menu .menu-item", self.say(f"{label}.", f"Choose {label.lower()}."), text=label, why=label) |
| return |
| if kind == "ppmenu": |
| label = {"left": "Align Left", "center": "Center", "right": "Align Right"}[align] |
| self.click(sel, self.say(f"Open Align to {label.lower()}.", "Align dropdown."), why="the Align button") |
| self.click(".pp-alignmenu .menu-item", self.say(f"{label}.", f"Choose {label}."), text=label, why=label) |
| return |
| if kind == "slpop": |
| self.click(sel, self.say(f"Open Align to align it {align}.", "Alignment dropdown."), why="the Align button") |
| self.click(f'.sl-pop [data-value="{align}"]', self.say(f"Align {align}.", f"{align.capitalize()}."), why=f"align {align}") |
| return |
| if kind == "cycle": |
| order = ["left", "center", "right"] |
| cur = self.js("window.__app.store.selected()[0]?.align || 'left'") |
| for _ in range((order.index(align) - order.index(cur)) % 3): |
| self.click(sel, self.say(f"Click alignment until it reads {align}.", f"Step the alignment to {align}."), |
| why="the alignment button") |
| return |
| self.click(sel.format(v=align), self.say(f"Align it {align}.", f"Set alignment to {align}."), |
| why="the alignment buttons") |
|
|
| def set_font(self, family, doc_fonts, button=".inspector .font-btn"): |
| st = self.style |
| self.ensure_for(button) |
| self.click(button, self.say(f"Open the font menu to choose {family}.", |
| f"The font should be {family}; open the font picker."), |
| why="the font menu") |
| row = f'.font-pop .font-row[data-family="{family}"]' |
| hits = self.locate(row) |
| if self.skin == "slides": |
| |
| |
| if hits: |
| return self.click(row, self.say(f"{family} is in the menu; pick it.", f"Click {family}."), why=f"{family} in the menu") |
| self.click(".sl-fontmenu .menu-item", self.say(f"Open More fonts to find {family}.", "More fonts."), text="More fonts", |
| why="More fonts") |
| self.turn(self.say(f"Search for {family}.", f"Type {family} in the search."), [("type", {"text": family})]) |
| self.click(f'.sl-fonts .font-row[data-family="{family}"]', self.say(f"Tick {family}.", f"Check {family}."), why=f"{family}") |
| return self.click(".sl-fonts .modal-foot .btn-primary", self.say("Done.", "Click Done."), why="Done") |
| in_doc = family in doc_fonts and hits and hits[0].get("visible", True) |
| if not (in_doc and st.chance(st.pick_doc_font)): |
| self.turn(self.say(f"Search for {family}.", f"Type {family} to filter the list."), |
| [("type", {"text": family})]) |
| self.click(row, self.say(f"Pick {family}.", f"Click {family} in the list."), why=f"{family} in the list") |
|
|
| def finish_common(self, el, what): |
| if el.angle and not (self.style.chance(self.style.drag_rotate) and self.rotate_by_drag(el, what)): |
| if self.skin == "inkscape": |
| self.rotate_inkscape(el, what) |
| else: |
| self.open("inspector") |
| self.set_field("angle", num(el.angle), thought=self.say( |
| f"It is rotated; set rotation to {num(el.angle)}.", f"Rotate it to {num(el.angle)} degrees.")) |
| if el.opacity < 1: |
| pop = POPOVER_FIELDS.get(self.skin, {}) |
| if pop.get("opacity"): |
| self.click(pop["opacity"], self.say("Open Transparency.", "Set the transparency."), why="the Transparency button") |
| self.set_field("opacity", num(round(el.opacity * 100)), kind="slider", |
| selector='.popover [data-field="opacity"] .range-num') |
| elif self.skin == "inkscape": |
| self.set_field("opacity", num(round(el.opacity * 100)), selector='.ik-status [data-field="opacity"] input') |
| elif self.skin == "powerpoint": |
| self.open("inspector") |
| self.pp_pane(top="text") if el.kind == "text" else self.pp_pane(sub="picture") |
| self.set_field("opacity", num(100 - round(el.opacity * 100)), kind="slider", |
| selector='.pp-format [data-field="opacity"] .range-num', |
| thought=self.say(f"Transparency {num(100 - round(el.opacity * 100))}%.", "Set the transparency.")) |
| elif self.skin == "slides": |
| self.open("inspector") |
| if not self.locate('.sl-fo-sec.open[data-sec="adjust"]'): |
| self.click('.sl-fo-sec[data-sec="adjust"] .sl-fo-head', self.say("Expand Adjustments for Transparency.", "Open Adjustments."), |
| why="Adjustments") |
| self.set_field("opacity", num(100 - round(el.opacity * 100)), kind="slider", |
| selector='.sl-format [data-field="opacity"] .range-num', |
| thought=self.say(f"Transparency {num(100 - round(el.opacity * 100))}%.", "Set the transparency.")) |
| elif self.skin == "figma": |
| self.open("inspector") |
| self.set_field("opacity", f"{num(round(el.opacity * 100))}%", selector='.inspector [data-field="opacity"] input') |
| else: |
| self.open("inspector") |
| self.set_field("opacity", num(round(el.opacity * 100)), kind="slider") |
| self.deselect() |
|
|
| |
|
|
| def selected_box(self): |
| return self.js("""(() => { const e = window.__app.store.selected()[0]; |
| return e ? {x: e.x, y: e.y, width: e.width, height: e.height, angle: e.angle || 0} : null; })()""") |
|
|
| def rotate_inkscape(self, el, what): |
| """Inkscape: Object > Transform, Rotate tab, the angle to turn by (relative), clockwise, Apply.""" |
| cur = self.selected_box()["angle"] |
| delta = el.angle - cur |
| if abs(delta) < 1e-6: |
| return |
| if not self.locate('.ik-docktab.on[data-tab="transform"]'): |
| self.click('[data-command="dlg_transform"]', self.say(f"Open Transform to rotate {what}.", "Transform (shift+ctrl+m)."), |
| why="Transform") |
| self.click('.ik-tr .ik-subtab[data-tab="rotate"]', self.say("Rotate tab.", "Go to Rotate."), why="the Rotate tab") |
| direction = "cw" if delta > 0 else "ccw" |
| if not self.locate(f'.ik-tr .ik-tr-dir button.on[data-value="{direction}"]'): |
| self.click(f'.ik-tr .ik-tr-dir button[data-value="{direction}"]', self.say( |
| "Rotate clockwise." if direction == "cw" else "Rotate counter-clockwise.", "Pick the direction."), why="the direction") |
| self.set_field("rotate_angle", num(abs(delta)), selector='.ik-tr [data-field="rotate_angle"] input', |
| thought=self.say(f"It turns by {num(abs(delta))} degrees.", f"Angle: {num(abs(delta))}.")) |
| self.click(".ik-tr .ik-apply", self.say("Apply the rotation.", "Apply."), why="the Apply button") |
|
|
| def ensure_handles(self, what): |
| """Photoshop shows handles only with Show Transform Controls, or in Free Transform (ctrl+t).""" |
| if self.skin != "photoshop" or self.locate(".overlay .sel-box .handle"): |
| return |
| st = self.style |
| box = [r for r in self.locate(".ps-options .ps-transform") if r.get("visible", True)] |
| if not box or st.chance(0.5): |
| self.turn(self.say(f"Free Transform (ctrl+t) to get handles on {what}.", "ctrl+t for the transform handles."), |
| [("key", {"keys": "ctrl+t"})]) |
| self._free_transform = True |
| else: |
| self.click(".ps-options .ps-transform", self.say("Turn on Show Transform Controls to get the handles.", |
| "Tick Show Transform Controls in the options bar."), why="Show Transform Controls") |
|
|
| def end_transform(self): |
| if getattr(self, "_free_transform", False): |
| self._free_transform = False |
| self.turn(self.say("Press enter to commit the transform.", "Commit the transform."), [("key", {"keys": "enter"})]) |
|
|
| def handle_at(self, name): |
| hits = self.locate(f".overlay .sel-box .handle.h-{name}") |
| return self.at(hits[0]) if hits and hits[0].get("visible", True) else None |
|
|
| def place(self, el, what, fields) -> bool: |
| """Size and position by direct manipulation: handle drag, body drag, arrow-key nudges. |
| |
| Returns False (nothing done) when the plain agent is running, the strategy |
| roll says "type the numbers", or the gesture cannot be made on screen. |
| """ |
| st = self.style |
| roll = st.rng.random() if st.rng else 1.0 |
| if roll >= st.drag_only + st.drag_refine: |
| return False |
| refine = roll >= st.drag_only |
| cur = self.selected_box() |
| if not cur or cur["angle"]: |
| return False |
| |
| if (el.kind == "image" and (abs(cur["width"] - el.width) > 0.5 or abs(cur["height"] - el.height) > 0.5)) or ( |
| el.kind == "text" and abs(cur["width"] - el.width) > 0.5): |
| self.ensure_handles(what) |
| if el.kind == "image" and (abs(cur["width"] - el.width) > 0.5 or abs(cur["height"] - el.height) > 0.5): |
| h, dst = self.handle_at("se"), self.canvas_point(cur["x"] + el.width, cur["y"] + el.height) |
| if not (h and dst): |
| return False |
| free = abs(cur["width"] / cur["height"] - el.width / el.height) > 0.01 |
| self.turn(self.say(f"Drag the corner handle to size {what} like the reference.", |
| f"Resize {what} from its bottom-right corner." |
| + (" Hold shift to change its proportions." if free else "")), |
| [("left_click_drag", {"start_coordinate": h, "coordinate": dst, **({"text": "shift"} if free else {})})]) |
| elif el.kind == "text" and abs(cur["width"] - el.width) > 0.5: |
| h, dst = self.handle_at("e"), self.canvas_point(cur["x"] + el.width, cur["y"] + cur["height"] / 2) |
| if not (h and dst): |
| return False |
| self.turn(self.say(f"Drag the right edge to make the {what[4:] if what.startswith('the ') else what} box wider.", |
| f"Set the width of {what} by dragging its side handle."), |
| [("left_click_drag", {"start_coordinate": h, "coordinate": [dst[0], h[1]]})]) |
| self.end_transform() |
| |
| cur = self.selected_box() |
| src = self.canvas_point(cur["x"] + cur["width"] / 2, cur["y"] + cur["height"] / 2) |
| dst = self.canvas_point(el.x + cur["width"] / 2, el.y + cur["height"] / 2) |
| if src and dst and abs(src[0] - dst[0]) + abs(src[1] - dst[1]) > 2: |
| self.turn(self.say(f"Drag {what} into place.", f"Move {what} to where it sits in the reference."), |
| [("left_click_drag", {"start_coordinate": src, "coordinate": dst})]) |
| elif not (src and dst): |
| return False |
| |
| cur = self.selected_box() |
| presses, words = [], [] |
| for d, neg, pos in ((el.x - cur["x"], "left", "right"), (el.y - cur["y"], "up", "down")): |
| n = round(abs(d)) |
| tens, ones = divmod(n, 10) |
| key = pos if d > 0 else neg |
| presses += [f"shift+{key}"] * tens + [key] * ones |
| if n: |
| words.append(f"{n} px {key}") |
| if 0 < len(presses) <= 14: |
| self.turn(self.say(f"Nudge it {' and '.join(words)} with the arrow keys.", |
| f"Almost there: {' and '.join(words)}."), [("key", {"keys": presses})]) |
| elif len(presses) > 14: |
| refine = True |
| |
| cur = self.selected_box() |
| |
| |
| size = min(el.width, el.height) |
| tol = 0.5 if refine else min(2.5, max(0.75, 0.01 * size)) |
| off = [f for f in fields if abs(cur[f] - getattr(el, f)) > tol] |
| if off: |
| self.open("inspector") |
| label = ", ".join(FIELD_NAME.get(f, f) for f in off) |
| self._pending_thought = self.say(f"The panel shows {label} still off; type the exact value.", |
| f"Fine-tune {label} in the properties panel.") |
| for f in off: |
| self.set_number(f, num(getattr(el, f)), what) |
| return True |
|
|
| def rotate_by_drag(self, el, what) -> bool: |
| """Turn the selected element with the rotate handle (Template) or a corner rotate zone.""" |
| import math |
|
|
| cur = self.selected_box() |
| if not cur or abs(el.angle) < 4: |
| return False |
| c = self.canvas_point(cur["x"] + cur["width"] / 2, cur["y"] + cur["height"] / 2) |
| self.ensure_handles(what) |
| if self.skin == "inkscape" and c and not self.locate(".overlay .sel-box .rot-zone"): |
| self.turn(self.say(f"Click {what} again for the rotate arrows.", "Click the selection to switch to rotate handles."), |
| [("left_click", {"coordinate": c})]) |
| hits = self.locate(".overlay .sel-box .rot-handle") or self.locate(".overlay .sel-box .rot-zone.z-ne") |
| if not (c and hits): |
| return False |
| hx, hy = hits[0]["x"], hits[0]["y"] |
| t = math.radians(el.angle - cur["angle"]) |
| vx, vy = hx - c[0], hy - c[1] |
| dst = [round(c[0] + vx * math.cos(t) - vy * math.sin(t)), round(c[1] + vx * math.sin(t) + vy * math.cos(t))] |
| if not (0 <= dst[0] < VIEW_W and 0 <= dst[1] < VIEW_H): |
| return False |
| self.turn(self.say(f"Rotate {what} by dragging the rotate handle.", f"Turn {what} to match the reference."), |
| [("left_click_drag", {"start_coordinate": [round(hx), round(hy)], "coordinate": dst})]) |
| self.end_transform() |
| got = self.selected_box()["angle"] |
| if abs(got - el.angle) > 0.3: |
| self.open("inspector") |
| self._pending_thought = self.say(f"Rotation reads {num(got)}; set it exactly.", "Fine-tune the angle.") |
| self.set_field("angle", num(el.angle)) |
| return True |
|
|
| def _order(self, items): |
| items = list(items) |
| if self.style.rng and self.style.shuffle_props: |
| self.style.rng.shuffle(items) |
| return items |
|
|
| def plan(self, gt, assets) -> str: |
| names = [self._describe(e, assets) for e in gt.elements] |
| listing = "; ".join(f"{i + 1}. {n}" for i, n in enumerate(names)) |
| return self.say( |
| f"The reference has {len(names)} layers, back to front: {listing}. I'll add them in that order " |
| "and type each one's exact position, size and style into the properties panel.", |
| f"Layers from the back: {listing}. Plan: place each layer, then set its numbers exactly.") |
|
|
| def build(self, gt, assets: dict | None = None): |
| assets = assets or {} |
| doc_fonts = {e.font_family for e in gt.elements if e.kind == "text"} |
| if self.rec is not None: |
| self._pending_thought = self.plan(gt, assets) |
| if gt.background.lower() != "#ffffff": |
| hexv = gt.background.lstrip("#")[:6] |
| if self.skin == "powerpoint": |
| if not self.locate('.pp-tabs [data-tab="design"].on'): |
| self.click('.pp-tabs [data-tab="design"]', self.say("Format Background is on the Design tab.", "Go to Design."), why="the Design tab") |
| if not self.locate('.pp-pane[data-panel="format"]'): |
| self.click('.pp-ribbon [data-open="background"]', self.say("Open Format Background.", "Format Background."), why="Format Background") |
| self.click('.pp-format [data-field="background"]', self.say(f"Open the fill Color for #{hexv}.", "Background colour."), why="the Color button") |
| self.pp_color(hexv) |
| elif self.skin in BG_ROUTES: |
| opener, done, field = BG_ROUTES[self.skin] |
| if not self.locate(done): |
| self.click(opener, self.say("Open Background to set the page colour.", "Background panel first.") |
| if self.skin == "canva" else |
| self.say("Open Background to set the slide colour.", "The Background button in the toolbar.") |
| if self.skin == "slides" else |
| self.say("Open Document Properties for the page's background colour.", "Document Properties (shift+ctrl+d) has the background.") |
| if self.skin == "inkscape" else |
| self.say("Select Frame 1 to set the background: it is the frame's fill.", |
| "The background is the frame's Fill; click Frame 1 in the layers."), |
| why=BG_NAME[self.skin]) |
| if self.skin == "inkscape": |
| self.click(field, self.say(f"Click the Background color swatch to set #{hexv}.", "Open the background colour."), |
| why="the Background color swatch") |
| self.turn(self.say(f"Type {hexv}ff into RGBA and press enter.", f"RGBA: {hexv}ff."), |
| [("key", {"keys": "ctrl+a"}), ("type", {"text": f"{hexv}ff"}), ("key", {"keys": "enter"}), ("key", {"keys": "escape"})]) |
| elif self.skin == "slides": |
| self.click(field, self.say(f"Open the Color menu for #{hexv}.", "Background colour menu."), why="the Color menu") |
| self.slides_color(hexv) |
| self.click(".modal.sl-bg .modal-foot .btn-primary", self.say("Done.", "Close with Done."), why="Done") |
| elif self.skin == "canva": |
| self.click(field, self.say(f"Open the background colour panel for #{hexv}.", "Click the palette for a custom colour."), |
| why="the palette button") |
| r = self.visible(".cv-colorpanel .cv-colsearch", why="the colour search") |
| self.turn(self.say(f"Search #{hexv} and press enter.", f"Type #{hexv}."), |
| [("left_click", {"coordinate": self.at(r)}), ("type", {"text": f"#{hexv}"}), ("key", {"keys": "enter"})]) |
| else: |
| self.set_field("page_background", hexv, kind="hex", selector=field, |
| thought=self.say(f"Set the background colour to #{hexv}.", f"Background: #{hexv}.")) |
| else: |
| self.open("inspector") |
| self.set_field("background", hexv, kind="hex", |
| thought=self.say(f"Set the background colour to {gt.background}.", |
| f"Background first: {gt.background}.")) |
| self.deselect() |
| st = self.style |
| for el in gt.elements: |
| if self.rec is not None and self.mode == "reference" and st.chance(st.view_reference): |
| self.turn(self.say(f"Look at the reference again for {self._describe(el, assets)}.", |
| f"Check the target: where does {self._describe(el, assets)} go?"), |
| [("view_reference", {})]) |
| if el.kind == "image": |
| self.add_image(el, assets) |
| else: |
| self.add_text(el, assets, doc_fonts) |
| self.deselect(self.say("Deselect and compare the canvas with the reference.", |
| "Everything is placed; check it against the reference.")) |
| if self.rec is not None and self.mode == "reference" and st.chance(2 * st.view_reference): |
| self.turn(self.say("Compare with the reference one last time.", "Final check against the target."), |
| [("view_reference", {})]) |
| if self.skin == "inkscape" and not self.locate('.ik-exportdlg [data-command="submit"]'): |
| self.click('[data-command="dlg_export"]', self.say("It matches the reference. Open Export.", "Done; File > Export (shift+ctrl+e)."), |
| why="Export") |
| self.click('[data-command="submit"]', self.say("It matches the reference. Export it.", |
| "The design matches; export it.", "Done. Click Export."), |
| why="the Export button") |
| return self.click(".modal-foot .btn-primary", self.say("Export as PNG.", "Confirm the export."), |
| why="the Export confirm button") |
|
|
|
|
| def _typo(value: str, rng: random.Random) -> str: |
| digits = [i for i, c in enumerate(value) if c.isdigit()] |
| if len(digits) >= 2 and rng.random() < 0.5: |
| i, j = digits[0], digits[1] |
| s = list(value) |
| s[i], s[j] = s[j], s[i] |
| if "".join(s) != value: |
| return "".join(s) |
| i = rng.choice(digits) |
| return value[:i] + str((int(value[i]) + rng.randrange(1, 9)) % 10) + value[i + 1:] |
|
|
|
|
| def _result_text(out) -> str: |
| obs = out.observation |
| if obs.error is not None: |
| return f"Error: {obs.error.message}" |
| res = obs.result or {} |
| text = "".join(c.get("text", "") for c in res.get("content", []) if c.get("type") == "text") |
| return text or str(res.get("data", "")) |
|
|
|
|
| |
|
|
|
|
| def look_for(task_id: str, seed: int, slot: int | None = None) -> dict: |
| """The editor look for one trajectory. |
| |
| With `slot` (the trajectory's position in a batch) the 12 looks (6 layouts x |
| 2 themes) are dealt out in turn, so any batch is balanced across layouts; the |
| file window's look (macOS, Windows, Linux) cycles alongside. |
| """ |
| rng = random.Random(int(hashlib.sha256(f"{seed}:{task_id}:look".encode()).hexdigest()[:12], 16)) |
| if slot is None: |
| look = {"skin": rng.choice(SKINS), "theme": rng.choice(THEMES), "layout": rng.choice(LAYOUTS)} |
| return {**look, "file_dialog": rng.choice(FILE_DIALOGS)} |
| looks = [(s, t, lay) for lay in LAYOUTS for t in THEMES for s in SKINS] |
| skin, theme, layout = looks[slot % len(looks)] |
| |
| dialog = FILE_DIALOGS[(slot + slot // len(looks)) % len(FILE_DIALOGS)] |
| return {"skin": skin, "theme": theme, "layout": layout, "file_dialog": dialog} |
|
|
|
|
| def record(server: str, pack, task: dict, out_dir: Path, *, mode: str = "reference", seed: int = 0, |
| look: dict | None = None, image_format: str = "webp", natural: bool = True, |
| min_reward: float = 0.98) -> dict: |
| """Record one verified trajectory. Raises if it scores below `min_reward`. |
| |
| Dragging leaves up to a few canvas pixels of residual where the recorder chose |
| not to type the exact number, so a natural trajectory scores 0.98-1.0, not 1.0. |
| """ |
| from openenv.core.mcp_client import MCPToolClient |
|
|
| look = look or look_for(task["task_id"], seed) |
| stem = f"{task['task_id']}__{mode}__{look['skin']}-{look['theme']}-{look['layout']}__s{seed}" |
| rng = random.Random(int(hashlib.sha256(f"{seed}:{task['task_id']}:{mode}".encode()).hexdigest()[:12], 16)) |
| rec = Recording(out_dir, stem, image_format) |
| gt = pack.gt_doc(task) |
| assets = {a["id"]: a for a in task["assets"]} |
| t0 = time.time() |
| with MCPToolClient(base_url=server, message_timeout_s=180).sync() as env: |
| r = env.reset(task_id=task["task_id"], surface="gui", mode=mode, max_steps=5000, seed=seed, **look) |
| meta = r.observation.metadata |
| tools = [t for t in env.list_tools(use_cache=False) if t.name != "start_task"] |
| agent = GuiAgent(server, env, meta["episode_id"], look["skin"], |
| style=Style(rng) if natural else Style(), recording=rec, mode=mode) |
| user = [] |
| if mode == "reference": |
| user.append({"type": "image", "image": rec.save_image( |
| base64.b64decode(meta["reference_jpeg_b64"]), "target")}) |
| user.append({"type": "image", "image": rec.save_image(agent.screenshot(), "t000")}) |
| intro = {"reference": "This is the target design. Recreate it.", |
| "description": "Build the design from the brief in your instructions."}[mode] |
| user.append({"type": "text", "text": intro + (" The second image is the editor as it is now." |
| if mode == "reference" else |
| " The image is the editor as it is now.")}) |
| rec.messages[:0] = [{"role": "system", "content": meta["instructions"]}, {"role": "user", "content": user}] |
| final = agent.build(gt, assets) |
| grade = final.observation.metadata.get("grade") if final.done else None |
| if grade is None: |
| out = env.step(CallToolAction(tool_name="submit", arguments={})) |
| grade = out.observation.metadata.get("grade") |
| reward = grade["total"] if grade else None |
| if reward is None or reward < min_reward: |
| worst = sorted(grade["elements"], key=lambda e: e["score"])[:3] if grade else [] |
| raise RuntimeError(f"{stem}: scored {reward}; weakest " |
| f"{[(e['gt_id'], round(e['score'], 3), e['note']) for e in worst]}") |
| return { |
| "messages": rec.messages, |
| "tools": [{"type": "function", "function": {"name": t.name, "description": t.description, |
| "parameters": t.input_schema}} for t in tools], |
| "images": rec.images, |
| |
| "meta": {"task_id": task["task_id"], "crello_id": task.get("crello_id"), |
| "task_source": "corpus" if type(pack).__name__ == "CorpusPack" else "pack", |
| "surface": "gui", "mode": mode, **look, "seed": seed, |
| "reward": reward, "turns": agent.turns, "actions": agent.actions, |
| "masked_turns": rec.masked, "mistakes": agent.mistakes, "n_elements": len(gt.elements), |
| "seconds": round(time.time() - t0, 1), "prompt_version": meta.get("prompt_version"), |
| "recorder_version": RECORDER_VERSION, "viewport": [VIEW_W, VIEW_H]}, |
| } |
|
|
|
|
| def start_debug_server() -> str: |
| """An in-process server with the locate hook enabled, on a free port.""" |
| import os |
| import socket |
|
|
| import uvicorn |
|
|
| os.environ["DESIGN_CANVAS_DEBUG"] = "1" |
| s = socket.socket() |
| s.bind(("127.0.0.1", 0)) |
| port = s.getsockname()[1] |
| s.close() |
| os.environ["PORT"] = str(port) |
| from .server.app import create_server |
|
|
| server = uvicorn.Server(uvicorn.Config(create_server(), host="127.0.0.1", port=port, log_level="warning", |
| ws_ping_interval=60, ws_ping_timeout=300)) |
| threading.Thread(target=server.run, daemon=True).start() |
| for _ in range(300): |
| try: |
| urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=1) |
| break |
| except Exception: |
| time.sleep(0.1) |
| return f"http://127.0.0.1:{port}" |
|
|
|
|
| def export(pack, split: str, out: Path, *, limit: int | None = None, start: int = 0, modes=("reference",), |
| seed: int = 0, workers: int = 2, server: str | None = None, max_elements: int = 12, |
| image_format: str = "webp", log=print) -> dict: |
| """Record trajectories for a split into `out/cua.jsonl` (+ images). Resumable.""" |
| from concurrent.futures import ThreadPoolExecutor |
|
|
| out = Path(out) |
| out.mkdir(parents=True, exist_ok=True) |
| server = server or start_debug_server() |
| done = set() |
| jsonl = out / "cua.jsonl" |
| if jsonl.exists(): |
| for line in jsonl.read_text().splitlines(): |
| m = json.loads(line)["meta"] |
| done.add((m["task_id"], m["mode"], m["seed"])) |
| jobs, skipped, slot = [], {}, 0 |
| for i in range(start, pack.count(split)): |
| if limit is not None and slot >= limit: |
| break |
| task = pack.at(split, i) |
| why = gui_expressible(pack, task) or ( |
| f"more than {max_elements} elements" if task["stats"]["n_elements"] > max_elements else None) |
| if why: |
| skipped[why] = skipped.get(why, 0) + 1 |
| continue |
| mode = modes[(i // len(SKINS)) % len(modes)] |
| if (task["task_id"], mode, seed) not in done: |
| jobs.append((task, mode, look_for(task["task_id"], seed, slot))) |
| slot += 1 |
| lock = threading.Lock() |
| stats = {"recorded": 0, "failed": 0, "turns": [], "actions": [], "images": 0} |
|
|
| def job(item): |
| task, mode, look = item |
| try: |
| traj = record(server, pack, task, out, mode=mode, seed=seed, look=look, image_format=image_format) |
| except Exception as err: |
| with lock: |
| stats["failed"] += 1 |
| with open(out / "failures.jsonl", "a") as fh: |
| fh.write(json.dumps({"task_id": task["task_id"], "mode": mode, "seed": seed, |
| "error": f"{type(err).__name__}: {err}"[:500]}) + "\n") |
| log(f" {task['task_id']} {mode}: FAILED {err}") |
| return |
| with lock: |
| with open(jsonl, "a") as fh: |
| fh.write(json.dumps(traj, ensure_ascii=False) + "\n") |
| m = traj["meta"] |
| stats["recorded"] += 1 |
| stats["turns"].append(m["turns"]) |
| stats["actions"].append(m["actions"]) |
| stats["images"] += len(traj["images"]) |
| log(f" {m['task_id']} {mode} {m['skin']}/{m['theme']}/{m['layout']}: reward {m['reward']:.3f}, " |
| f"{m['turns']} turns, {m['actions']} actions, {len(m['mistakes'])} slips, {m['seconds']}s") |
|
|
| log(f"recording {len(jobs)} trajectories with {workers} workers -> {out}") |
| with ThreadPoolExecutor(workers) as ex: |
| list(ex.map(job, jobs)) |
| |
| metas = [json.loads(line)["meta"] for line in jsonl.read_text().splitlines()] if jsonl.exists() else [] |
| summary = {"split": split, "seed": seed, "recorded": len(metas), "failed_this_run": stats["failed"], |
| "skipped": skipped, "images": sum(1 for _ in (out / "images").rglob("*.*")) if (out / "images").exists() else 0, |
| "mean_turns": round(sum(m["turns"] for m in metas) / max(1, len(metas)), 1), |
| "mean_actions": round(sum(m["actions"] for m in metas) / max(1, len(metas)), 1), |
| "mean_reward": round(sum(m["reward"] for m in metas) / max(1, len(metas)), 4), |
| "min_reward": round(min((m["reward"] for m in metas), default=0.0), 4)} |
| (out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") |
| return summary |
|
|
