"""Computer-use (CUA) trajectories: rebuild a design with mouse and keyboard, recorded for SFT. A scripted agent drives the GUI surface through the environment's own tools (`left_click`, `type`, `key`, `left_click_drag`, `scroll` over MCP), exactly as a model would, and each turn is recorded the way the rollout harness presents it to a model: a short thought, one or a few tool calls, then a fresh screenshot. The recording is only kept if the submitted design scores 1.0, so every trajectory is a verified solution. It is built to look like a competent agent, not a macro: - **Grounded.** Every coordinate is where the control is on the *current* screen. The only thing the script gets for free is that lookup (a debug hook, `DESIGN_CANVAS_DEBUG=1`); the recording keeps only the coordinates, so a model trained on it has to find controls from pixels. - **Looks, then acts.** One turn = one intention: open a panel, place an asset, set a field (click, select, type, commit). A screenshot follows every turn, and a field below the fold is scrolled into view first. - **Images come from files.** Each layout's own route opens the computer's file window (Studio's Shape tools > Image/video..., Pixel's File > Place Embedded..., Deck's Insert > Upload from computer, Present's Insert > Pictures > This Device..., Vector's File > Import..., Template's Uploads > Upload files). The file is then opened (Open, double-click or Enter) or dragged out of the window onto the canvas, moving or shrinking the window first when it covers the spot. Studio then places by click or by dragging out the box; Vector asks how to import; Template lists uploads, which are clicked in or dragged to place. - **Varied gestures, seeded.** Text is placed by clicking near its spot; fields are selected by triple-click or ctrl+a and committed by enter or tab; property order varies; fonts are searched or picked from "Fonts in this design". - **Mistakes and recovery.** A few turns contain a realistic slip (a typo in a field, the wrong asset) followed by noticing and fixing it. The slip turn is marked `"train": false` so a trainer can mask it and learn only the recovery. - **Every look.** All three layouts (Studio, Template, Pixel) in light and dark, dealt out evenly across a batch. The target is never visible in the editor; the agent has it in its first message and can call `view_reference`. Same task + seed gives the same trajectory. design-canvas export-cua --split train --limit 200 --out artifacts/cua """ from __future__ import annotations import base64 import hashlib import io import json import random import threading import time import urllib.request from dataclasses import dataclass, field from pathlib import Path from openenv.core.env_server.mcp_types import CallToolAction from .server.environment import FILE_DIALOGS from .server.episodes import SKINS # every editor layout VIEW_W, VIEW_H = 1280, 800 THEMES = ("light", "dark") LAYOUTS = ("default",) # mirrored panels ("swap") are opt-in via look=; real editors never flip RECORDER_VERSION = 3 # 2: images through the file window; 3: picked from the Open dialog, not dragged out of it # How to reach a panel in each layout. A string is one click; a list is a path of # (selector, already-there selector) steps, e.g. PowerPoint's Insert tab, then Pictures. OPENERS = { "assets": {"canva": '.cv-rail-btn[data-panel="uploads"]'}, # uploaded files; the other layouts use the file window "inspector": {"figma": '.fg-tabs [data-tab="design"]', "canva": [('.cv-context [data-open="position"]', '.cv-editpanel[data-panel="position"]'), ('.cv-pos-tabs [data-tab="arrange"]', '.cv-pos-tabs [data-tab="arrange"].on')], "photoshop": '.ps-tabs [data-tab="properties"]', "slides": '.sl-toolbar [data-open="format"]', "powerpoint": [('.pp-tabs [data-tab="format"]', '.pp-tabs [data-tab="format"].on'), ('.pp-ribbon [data-open="format"]', None)], "inkscape": '[data-command="toolMove"]'}, # the Selector's controls hold X / Y / W / H } STEP_NAME = {'.pp-tabs [data-tab="insert"]': "the Insert tab", '.pp-tabs [data-tab="home"]': "the Home tab", '.pp-tabs [data-tab="format"]': "the Shape Format tab", '.pp-tabs [data-tab="design"]': "the Design tab", '.cv-context [data-open="position"]': "Position in the toolbar", '.cv-pos-tabs [data-tab="arrange"]': "the Arrange tab"} # Where the page background colour is set when it is not in the properties panel. BG_ROUTES = {"canva": ('.cv-rail-btn[data-panel="background"]', '.cv-rail-btn[data-panel="background"].on', ".cv-bgpanel .cv-bgpalette"), # Figma: the background is the frame's Fill; select the frame (its layer row) first. "figma": ('.layers .fg-frame-row', '.inspector .fg-frame-head', '.inspector .fg-frame [data-field="background"] .hex-input'), # Inkscape: File > Document Properties, then the Background color swatch's RGBA entry. "inkscape": ('[data-command="dlg_document"]', '.ik-docktab.on[data-tab="document"]', ".ik-dp .ik-bgchip"), # Slides: the toolbar's Background button (Slide > Change background), then its Color menu. "slides": (".sl-toolbar .sl-bgbtn", ".modal.sl-bg", ".modal.sl-bg .sl-bgchip")} BG_NAME = {"canva": "the Background tab", "figma": "Frame 1 in the layers", "inkscape": "Document Properties", "slides": "the Background button"} ACTIVE = { # a selector that exists only when the panel is already open "assets": {"canva": '.cv-rail-btn[data-panel="uploads"].on'}, "inspector": {"canva": '.cv-editpanel[data-panel="position"] .cv-pos-tabs [data-tab="arrange"].on', "figma": '.fg-tabs [data-tab="design"].on', "photoshop": '.ps-tabs [data-tab="properties"].on', "slides": '.sl-side[data-panel="format"]', "powerpoint": '.pp-pane[data-panel="format"]', "inkscape": '.ik-tool.on[data-command="toolMove"]'}, } PANEL_NAME = {"assets": {"canva": "Uploads"}, "inspector": {"figma": "Design", "canva": "Position", "photoshop": "Properties", "slides": "Format options", "powerpoint": "Format", "inkscape": "Selector"}} # How each layout opens the computer's file window, as the real app does. Each route is a list of # steps (selector, exact text or None, already-there selector or None, what the step is called); # a layout with several routes gets one at random. SHORTCUT is the real app's key for it. PLACE_ITEM = '.menu [data-command="placeImage"]' FILE_ROUTES = { "figma": [[('.fg-caret[title="Shape tools"]', None, None, "the Shape tools menu"), (PLACE_ITEM, None, None, "Image/video…")]], "canva": [[('.cv-rail-btn[data-panel="uploads"]', None, '.cv-rail-btn[data-panel="uploads"].on', "the Uploads tab"), (".cv-upload", None, None, "Upload files")]], "photoshop": [[(".ps-menu", "File", None, "the File menu"), (PLACE_ITEM, None, None, "Place Embedded…")]], "slides": [[(".sl-menu", "Insert", None, "the Insert menu"), (".pop-menu .menu-item.has-sub", "Image", None, "Image"), (PLACE_ITEM, None, None, "Upload from computer")], [('.sl-toolbar [data-open="images"]', None, None, "the Insert image button"), (PLACE_ITEM, None, None, "Upload from computer")]], "powerpoint": [[('.pp-tabs [data-tab="insert"]', None, '.pp-tabs [data-tab="insert"].on', "the Insert tab"), ('.pp-ribbon [data-open="pictures"]', None, None, "Pictures"), (PLACE_ITEM, None, None, "This Device…")]], "inkscape": [[(".ik-menu", "File", None, "the File menu"), (PLACE_ITEM, None, None, "Import…")], [('.ik-commands [data-open="files"]', None, None, "the Import button")]], } SHORTCUT = {"figma": "ctrl+shift+k", "inkscape": "ctrl+i"} FILE_WINDOW = {"mac": "Finder window", "windows": "Open dialog", "linux": "file chooser"} FIELD_NAME = {"x": "X", "y": "Y", "width": "W", "height": "H", "angle": "rotation", "font_size": "font size", "line_height": "line height", "letter_spacing": "letter spacing", "color": "text colour", "background": "background colour", "opacity": "opacity"} # Where each layout's own text controls live. Studio styles text in its side # panel; Template has a contextual toolbar over the canvas and Pixel an options # bar under the menu, which is where people (and models) reach first. TEXT_ROUTES = { "panel": {"font": ".inspector .font-btn", "size": '.inspector [data-field="font_size"] input', "toggle": '.inspector button[title="{t}"]', "color": ("hex", '.inspector [data-field="color"] .hex-input'), "align": ("segmented", '.inspector .segmented button[data-value="{v}"]')}, "canva": {"font": ".cv-context .cv-font", "size": ".cv-context .cv-size input", "toggle": '.cv-context button[title="{t}"]', "color": ("panel", ".cv-context .cv-textcolor"), "align": ("cycle", '.cv-context button[title="Alignment"]')}, "photoshop": {"font": ".ps-options .ps-font", "size": '.ps-options input[aria-label="Font size"]', "toggle": '.inspector button[title="{t}"]', "color": ("popover", ".ps-options .ps-color .color-chip"), "align": ("segmented", '.ps-options .segmented button[data-value="{v}"]')}, # Slides: font menu (More fonts for the rest), text colour menu (CUSTOM +), the align dropdown; # capitals in Format > Text > Capitalization, spacing in Line & paragraph spacing > Custom spacing. "slides": {"font": ".sl-toolbar .sl-font", "size": ".sl-toolbar .sl-size input", "toggle": '.sl-toolbar button[title="{t}"]', "color": ("slmenu", ".sl-toolbar .sl-textcolor"), "align": ("slpop", ".sl-toolbar .sl-align")}, # PowerPoint: the Home tab's font / size comboboxes, B I U, the Font Color gallery and the Align dropdown; # All Caps and character spacing in the Font dialog, line spacing in Line Spacing / Paragraph. "powerpoint": {"font": ".pp-ribbon .pp-font", "size": ".pp-ribbon .pp-size input", "toggle": '.pp-ribbon button[title="{t}"]', "color": ("ppgallery", ".pp-ribbon .pp-textcolor-drop"), "align": ("ppmenu", ".pp-ribbon .pp-align"), "pre": ('.pp-tabs [data-tab="home"]', '.pp-tabs [data-tab="home"].on')}, # Figma: weight is a dropdown, underline and case sit behind Type settings (see style_figma). "figma": {"font": ".inspector .font-btn", "size": '.inspector [data-field="font_size"] input', "toggle": None, "color": ("hex", '.inspector [data-field="color"] .hex-input'), "align": ("segmented", '.inspector .segmented button[data-value="{v}"]'), "pre": ('.fg-tabs [data-tab="design"]', '.fg-tabs [data-tab="design"].on')}, # Inkscape: the Text tool's controls (font, style, size, line height, alignment, letter spacing); # colour in Fill and Stroke's RGBA entry; underline / capitals in Text and Font > Features. "inkscape": {"font": ".ik-controls .ik-font", "size": ".ik-controls .ik-size input", "toggle": None, "color": ("ikfill", '[data-command="dlg_fill"]'), "align": ("ikmenu", ".ik-controls .ik-align"), "pre": ('[data-command="toolText"]', '.ik-tool.on[data-command="toolText"]')}, } # Layouts whose properties panel has no typography: text is always styled from their toolbar, # and spacing / transparency come from the toolbar's own popovers. ALWAYS_NATIVE = {"canva", "figma", "inkscape", "slides", "powerpoint"} POPOVER_FIELDS = { "canva": {"spacing": '.cv-context button[title="Spacing"]', "opacity": '.cv-context button[title="Transparency"]'}, } TOOLBAR_NAME = {"canva": "the text toolbar", "photoshop": "the options bar", "slides": "the toolbar", "powerpoint": "the Home ribbon", "inkscape": "the tool controls bar", "panel": "the properties panel", "figma": "the Typography section"} def num(v: float) -> str: r = round(float(v), 2) return str(int(r)) if r == int(r) else str(r) def gui_expressible(pack, task) -> str | None: """Why the GUI cannot reproduce this task exactly, or None if it can.""" gt = pack.gt_doc(task) if any(e.runs for e in gt.elements): return "mixed-style text runs are not editable in the GUI" if any(e.kind == "shape" for e in gt.elements): return "shape elements" return None # ================================================================== the agent @dataclass class Style: """How this episode's agent behaves. `None` rng = the plain, fixed behaviour.""" rng: random.Random | None = None drag_assets: float = 0.5 # drag the tile onto the canvas instead of clicking it # An app's Open dialog is modal: files are picked from it (Open, double-click), never dragged out of it # onto the page. Dragging belongs to a separate Finder / Explorer window, which no route opens yet. drag_files: float = 0.0 shortcut: float = 0.3 # open the file window with the app's shortcut, where it has one open_by: tuple = (0.45, 0.35) # choose a file then Open | double-click it (otherwise select + Enter) place_box: float = 0.5 # Studio: drag out the image's box rather than click to place resize_window: float = 0.4 # shrink the file window out of the way rather than move it upload_all: float = 0.5 # Template: upload every image at once place_text: float = 0.7 # click near the text's spot instead of the canvas centre triple_click: float = 0.7 # select a field by triple-click instead of click + ctrl+a tab_commit: float = 0.45 # commit with tab instead of enter one_per_turn: float = 0.4 # field edit as select | type | commit, a screenshot after each two_per_turn: float = 0.25 # select | type + commit (otherwise all three in one turn) observe: float = 0.3 # open the next thought with what the last screenshot showed drag_text: float = 0.3 # drag a new text box roughly into place before typing numbers click_away: float = 0.4 # deselect by clicking empty canvas instead of pressing escape # How an element gets its size and place. Designers mostly drag handles and the # body, nudge with the arrow keys, and type a number only when something is off. drag_only: float = 0.5 # drag + nudge, accept the few-pixel residual drag_refine: float = 0.35 # drag + nudge, then type any field still off (else: type only) drag_rotate: float = 0.6 # rotate with the handle rather than the rotation field pick_doc_font: float = 0.5 # pick from "Fonts in this design" without typing native_text: float = 0.7 # style text with the layout's own toolbar rather than the side panel view_reference: float = 0.25 # look at the reference again before a layer (and before submitting) mistake: float = 0.12 # chance of a slip on a given element max_mistakes: int = 2 shuffle_props: bool = True def chance(self, p: float) -> bool: return bool(self.rng) and self.rng.random() < p class Say: """Short, varied thoughts. Only used when recording.""" def __init__(self, rng: random.Random | None): self.rng = rng def __call__(self, *options: str) -> str: return self.rng.choice(options) if self.rng else options[0] @dataclass class Recording: """What the agent did, in the rollout harness's message shape.""" out_dir: Path stem: str image_format: str = "webp" messages: list[dict] = field(default_factory=list) images: list[str] = field(default_factory=list) n_calls: int = 0 masked: int = 0 def save_image(self, data: bytes, name: str) -> str: from PIL import Image rel = f"images/{self.stem}/{name}.{self.image_format}" path = self.out_dir / rel path.parent.mkdir(parents=True, exist_ok=True) if self.image_format == "png": path.write_bytes(data) else: img = Image.open(io.BytesIO(data)).convert("RGB") img.save(path, "WEBP", quality=90, method=4) if self.image_format == "webp" else \ img.save(path, "JPEG", quality=90) self.images.append(rel) return rel class GuiAgent: """Rebuilds a ground-truth document through the GUI tools only.""" def __init__(self, server: str, env, episode_id: str, skin: str, *, style: Style | None = None, recording: Recording | None = None, verbose: bool = False, mode: str = "reference"): self.server = server.rstrip("/") self.mode = mode self.env = env self.eid = episode_id self.skin = skin self.style = style or Style() self.say = Say(self.style.rng) self.rec = recording self.verbose = verbose self.actions = 0 self.turns = 0 self.last = None self.mistakes: list[dict] = [] self._pending_thought = "" # ------------------------------------------------------------ plumbing def _post(self, path, body): req = urllib.request.Request(self.server + path, data=json.dumps(body).encode(), headers={"content-type": "application/json"}) with urllib.request.urlopen(req, timeout=30) as r: return json.loads(r.read()) def locate(self, selector, text=None): return self._post(f"/api/debug/gui/{self.eid}/locate", {"selector": selector, "text": text}) def js(self, code): return self._post(f"/api/debug/gui/{self.eid}/eval", {"js": code}) def _exec(self, tool, args): out = self.env.step(CallToolAction(tool_name=tool, arguments=args)) self.actions += 1 self.last = out if out.observation.error is not None: raise RuntimeError(f"{tool}{args}: {out.observation.error.message}") if self.verbose: print(f" {tool} {args}") return out def screenshot(self) -> bytes: out = self.env.step(CallToolAction(tool_name="screenshot", arguments={})) block = (out.observation.result or {}).get("content", [{}])[0] return base64.b64decode(block["data"]) def turn(self, thought: str, calls: list[tuple[str, dict]], train: bool = True): """Execute one turn; when recording, append it and the screenshot that follows.""" if self._pending_thought: thought = f"{self._pending_thought} {thought}".strip() self._pending_thought = "" results = [self._exec(tool, args) for tool, args in calls] self.turns += 1 if self.rec is None: return results[-1] ids = [f"call_{self.rec.n_calls + i}" for i in range(len(calls))] self.rec.n_calls += len(calls) msg = {"role": "assistant", "content": thought, "tool_calls": [ {"id": cid, "type": "function", "function": {"name": tool, "arguments": json.dumps(args, ensure_ascii=False)}} for cid, (tool, args) in zip(ids, calls, strict=True)]} if not train: msg["train"] = False self.rec.masked += 1 self.rec.messages.append(msg) done = results[-1].done for i, (cid, (tool, _), out) in enumerate(zip(ids, calls, results, strict=True)): text = _result_text(out) parts = [{"type": "text", "text": text}] if text else [] for k, block in enumerate((out.observation.result or {}).get("content", [])): if block.get("type") == "image": # e.g. view_reference returns the target rel = self.rec.save_image(base64.b64decode(block["data"]), f"t{self.turns:03d}-{tool}{k}") parts.append({"type": "image", "image": rel}) if i == len(calls) - 1 and not done: rel = self.rec.save_image(self.screenshot(), f"t{self.turns:03d}") parts.append({"type": "image", "image": rel}) content: list | str = parts if any(p["type"] == "image" for p in parts) else text self.rec.messages.append({"role": "tool", "tool_call_id": cid, "name": tool, "content": content}) return results[-1] # ------------------------------------------------------------ gestures def visible(self, selector, text=None, why=""): """Where to click a control as it is on screen now, scrolling its panel into view first. "Visible" means what a person would see: not clipped by its scrolling panel, and the control itself is under the pointer (not a popover or the menu bar). Each scroll is its own turn, with a screenshot, as it would be for a model. """ for _ in range(12): hits = self.locate(selector, text) if not hits: raise RuntimeError(f"not on screen: {selector} {text or ''}") r = hits[0] if r.get("visible", True) and 0 <= r["x"] < VIEW_W and 0 <= r["y"] < VIEW_H: return r sc = r.get("scroller") if not sc: raise RuntimeError(f"not reachable ({'covered' if r.get('covered') else 'off screen'}): " f"{selector} at {self.at(r)}") what = why or "it" self.turn(self.say(f"{what[0].upper() + what[1:]} is out of view; scroll the panel {sc['dir']}.", f"Scroll {sc['dir']} to bring {what} into view."), [("scroll", {"coordinate": [round(sc["x"]), round(sc["y"])], "scroll_direction": sc["dir"], "scroll_amount": 2})]) raise RuntimeError(f"could not scroll to {selector}") @staticmethod def at(r) -> list[int]: return [round(r["x"]), round(r["y"])] def click(self, selector, thought, text=None, clicks=1, why="", train=True): r = self.visible(selector, text, why) tool = {1: "left_click", 2: "double_click", 3: "triple_click"}[clicks] return self.turn(thought, [(tool, {"coordinate": self.at(r)})], train=train) def canvas_point(self, x, y): """Screen position of a canvas point, or None when it is off the visible stage.""" p = self.js(f"""(() => {{ const s = window.__app.stage, r = s.el.getBoundingClientRect(); const [sx, sy] = s.toScreen({x}, {y}); const X = r.left + sx, Y = r.top + sy; return (X > r.left + 4 && X < r.right - 4 && Y > r.top + 4 && Y < r.bottom - 4) ? [X, Y] : null; }})()""") return [round(p[0]), round(p[1])] if p else None def open(self, panel): active = ACTIVE[panel].get(self.skin) if active and self.locate(active): if self.skin == "powerpoint" and panel == "inspector": self.pp_pane(top="shape", sub="size") return name = PANEL_NAME[panel][self.skin] path = OPENERS[panel][self.skin] steps = [(path, None)] if isinstance(path, str) else path for k, (sel, done) in enumerate(steps): if done and self.locate(done): continue last = k == len(steps) - 1 where = STEP_NAME.get(sel, f"the {name} tab") self.click(sel, self.say(f"Open the {name} panel.", f"Switch to the {name} panel.", f"The {name} panel has what I need; open it.") if last else self.say(f"Go to {where}.", f"Open {where} to find {name}."), why=where) if self.skin == "powerpoint" and panel == "inspector": self.pp_pane(top="shape", sub="size") def ensure_for(self, selector): """A control inside the properties panel needs that panel open first.""" if selector and selector.lstrip().startswith(".inspector"): self.open("inspector") def set_field(self, name, value, kind="num", thought=None, train=True, selector=None): sel = selector or {"num": f'.inspector [data-field="{name}"] input', "hex": f'.inspector [data-field="{name}"] .hex-input', "slider": f'.inspector [data-field="{name}"] .range-num'}[kind] label = FIELD_NAME.get(name, name.replace("_", " ")) self.ensure_for(sel) r = self.visible(sel, why=f"the {label} field") st = self.style select = ([("triple_click", {"coordinate": self.at(r)})] if st.chance(st.triple_click) else [("left_click", {"coordinate": self.at(r)}), ("key", {"keys": "ctrl+a"})]) commit = ("key", {"keys": "tab" if st.chance(st.tab_commit) else "enter"}) shown = f"#{value}" if kind == "hex" else value thought = thought or self.say(f"Set {label} to {shown}.", f"{label[0].upper() + label[1:]}: {shown}.", f"Type {shown} into the {label} field.") # Strong models mostly act once per screenshot; sometimes they chain the # whole edit. Follow-up turns of one edit carry no new thought, as theirs don't. roll = st.rng.random() if st.rng else 1.0 if roll < st.one_per_turn: self.turn(thought, select, train=train) self.turn("", [("type", {"text": value})], train=train) out = self.turn("", [commit], train=train) elif roll < st.one_per_turn + st.two_per_turn: self.turn(thought, select, train=train) out = self.turn("", [("type", {"text": value}), commit], train=train) else: out = self.turn(thought, [*select, ("type", {"text": value}), commit], train=train) if train and st.chance(st.observe): self._pending_thought = self.say(f"{label[0].upper() + label[1:]} now reads {shown}.", f"{label[0].upper() + label[1:]} is {shown}.", "Good.") return out def set_number(self, name, value, what, kind="num", selector=None): """A field, sometimes with a realistic slip first (masked) and then the fix.""" st = self.style if (st.rng and len(self.mistakes) < st.max_mistakes and st.chance(st.mistake / 3) and kind == "num" and len(value.replace(".", "").replace("-", "")) >= 2): wrong = _typo(value, st.rng) self.set_field(name, wrong, kind, train=False, selector=selector) self.mistakes.append({"kind": "typo", "field": name, "typed": wrong, "meant": value}) label = FIELD_NAME.get(name, name) should = {"x": f"should be at x {value}", "y": f"should be at y {value}", "width": f"should be {value} wide", "height": f"should be {value} tall", "font_size": f"should be {value} px"}.get(name, f"needs {value}") self._pending_thought = self.say( f"The {label} field shows {wrong}, but {what} {should}; fix it.", f"That's a typo: {label} reads {wrong}. Correct it to {value}.") self.set_field(name, value, kind, selector=selector) def toggle(self, title, want: bool, template='.inspector button[title="{t}"]'): sel = template.format(t=title) probe = f"""(() => {{ const b = [...document.querySelectorAll('{sel}')].find((e) => e.offsetParent !== null); return b ? b.classList.contains('on') : null; }})()""" self.ensure_for(sel) state = self.js(probe) if state is None and not sel.startswith(".inspector"): # Not in this layout's toolbar (e.g. Uppercase): use the properties panel. return self.toggle(title, want) if state is None: raise RuntimeError(f"no {title} toggle on screen") if state is not None and state != want: self.click(sel, self.say(f"Turn {title.lower()} {'on' if want else 'off'}.", f"{title} should be {'on' if want else 'off'}."), why=f"the {title} button") def deselect(self, thought=None): st = self.style if not st.rng: # plain agent: the original double escape return self.turn(thought or "Deselect.", [("key", {"keys": "escape"}), ("key", {"keys": "escape"})]) spot = self.empty_canvas() if st.chance(st.click_away) else None if spot: return self.turn(thought or self.say("Click empty canvas to deselect.", "Deselect by clicking off the design."), [("left_click", {"coordinate": spot})]) self.turn(thought or self.say("Deselect before the next layer.", "Clear the selection.", "Press escape to deselect."), [("key", {"keys": "escape"})]) if self.js("window.__app.store.selection.length"): self.turn("", [("key", {"keys": "escape"})]) def empty_canvas(self): """A point on the grey stage around the artboard that hits nothing (or None).""" p = self.js("""(() => { const st = window.__app.stage.el, r = st.getBoundingClientRect(); const ab = document.querySelector('.artboard').getBoundingClientRect(); const pts = [[r.left + 24, r.top + 24], [r.right - 24, r.top + 24], [r.left + 24, (r.top + r.bottom) / 2], [r.right - 24, (r.top + r.bottom) / 2], [(ab.left + ab.right) / 2, ab.top - 14]]; for (const [x, y] of pts) { if (x > ab.left - 4 && x < ab.right + 4 && y > ab.top - 4 && y < ab.bottom + 4) continue; const at = document.elementFromPoint(x, y); if (at && (at === st || at.classList.contains('stage-inner') || at.classList.contains('stage'))) return [x, y]; } return null; })()""") return [round(p[0]), round(p[1])] if p else None # ------------------------------------------------------------ building def _describe(self, el, assets) -> str: if el.kind == "text": line = el.text.split("\n")[0] return f"the text “{line[:40]}{'…' if len(line) > 40 else ''}”" kind = assets.get(el.asset_id, {}).get("kind", "image") return {"photo": f"the photo {el.asset_id}", "background": f"the background image {el.asset_id}", "graphic": f"the graphic {el.asset_id}"}.get(kind, f"asset {el.asset_id}") def add_image(self, el, assets): what = self._describe(el, assets) images = list(assets) if self.skin == "canva" and self.js(f"window.__app.uploaded?.has({json.dumps(el.asset_id)})"): self.from_uploads(el, what, assets) else: self.from_files(el, what, images) if not self.place(el, what, ("width", "height", "x", "y")): self.open("inspector") self.set_number("width", num(el.width), what) self.set_number("height", num(el.height), what) for name in self._order(["x", "y"]): self.set_number(name, num(getattr(el, name)), what) self.finish_common(el, what) # ------------------------------------------------------------ images: the file window def chooser_os(self): return self.js("document.querySelector('.fc')?.dataset.os || null") def open_file_window(self, what): """Open the computer's file window the way this layout does it.""" st = self.style if self.chooser_os(): return if self.skin in SHORTCUT and st.chance(st.shortcut): keys = SHORTCUT[self.skin] self.turn(self.say(f"Open the file window with {keys} to add {what}.", f"{keys} opens the file picker; I need {what}."), [("key", {"keys": keys})]) else: route = st.rng.choice(FILE_ROUTES[self.skin]) if st.rng else FILE_ROUTES[self.skin][0] for k, (sel, text, done, name) in enumerate(route): if done and self.locate(done): continue first, last = k == 0, k == len(route) - 1 thought = (self.say(f"To add {what} I need its file: open {name}.", f"Images come from files here; open {name}.") if first else self.say(f"Choose {name}.", f"Click {name}.", f"{name} opens the file window.") if last else self.say(f"Then {name}.", f"Open {name}.")) self.click(sel, thought, text=text, why=name) if not self.chooser_os(): raise RuntimeError(f"{self.skin}: the file window did not open") def file_item(self, asset_id): return f'.fc .fc-item[data-file="{asset_id}"]' def from_files(self, el, what, images): st = self.style self.open_file_window(what) win = FILE_WINDOW.get(self.chooser_os(), "file window") fname = self.js(f"document.querySelector('{self.file_item(el.asset_id)}')?.title") if fname is None: raise RuntimeError(f"{el.asset_id} is not in the file window") canva = self.skin == "canva" pending = [a for a in images if not self.js(f"window.__app.uploaded?.has({json.dumps(a)})")] if canva else [] if canva and len(pending) > 1 and st.chance(st.upload_all): return self.upload_all(el, what, pending, win) target = self.canvas_point(el.x + el.width / 2, el.y + el.height / 2) if target and st.chance(st.drag_files) and self.uncover(target, what, win, el.asset_id): self.maybe_wrong_file(el, images, what) item = self.visible(self.file_item(el.asset_id), why=fname) self.turn(self.say(f"Drag {fname} out of the {win} onto the canvas where {what} sits.", f"Drop {fname} straight onto the canvas, roughly in place."), [("left_click_drag", {"start_coordinate": self.at(item), "coordinate": target})]) self.confirm_import() self.close_file_window(win) # a dropped file is on the page (and, in Template, in Uploads) return self.maybe_wrong_file(el, images, what) self.open_file(el.asset_id, fname, what, win) self.confirm_import() if self.skin == "figma": self.click_to_place(el, what) elif canva: self.from_uploads(el, what, None, fresh=True) def maybe_wrong_file(self, el, images, what): """Sometimes click the wrong file first (masked), then the right one.""" st = self.style if not (st.rng and len(self.mistakes) < st.max_mistakes and len(images) > 1 and st.chance(st.mistake / 2)): return wrong = st.rng.choice([a for a in images if a != el.asset_id]) name = self.js(f"document.querySelector('{self.file_item(wrong)}')?.title") if not name: return self.click(self.file_item(wrong), self.say(f"Select the file for {what}.", "Pick the image file."), why=name, train=False) self.mistakes.append({"kind": "wrong_file", "selected": wrong, "meant": el.asset_id}) self._pending_thought = self.say(f"That selected {name}; {what} is {el.asset_id}.", f"Wrong file: {name} is highlighted, but I need {el.asset_id}.") def open_file(self, asset_id, fname, what, win): st = self.style item = self.file_item(asset_id) roll = st.rng.random() if st.rng else 0.0 if roll < st.open_by[0]: self.click(item, self.say(f"Select {fname}.", f"Click {fname} in the {win}."), why=fname) label = self.js("document.querySelector('.fc [data-fc=\"open\"]').textContent") self.click('.fc [data-fc="open"]', self.say("Open it.", f"Click {label}."), why="the Open button") elif roll < st.open_by[0] + st.open_by[1]: self.click(item, self.say(f"Double-click {fname} to open it.", f"Open {fname} with a double-click."), clicks=2, why=fname) else: self.click(item, self.say(f"Select {fname}.", f"Click {fname}."), why=fname) self.turn(self.say("Press enter to open it.", "Enter opens the selected file."), [("key", {"keys": "enter"})]) if self.chooser_os(): raise RuntimeError(f"{fname} did not open") def upload_all(self, el, what, pending, win): """Template: select every image in the window and upload them together.""" st = self.style if st.chance(0.5): first = self.visible(self.file_item(pending[0]), why="the first file") self.turn(self.say("Upload all the images at once: select each file, then Open.", "Select every image file and upload them together."), [("left_click", {"coordinate": self.at(first)})]) for a in pending[1:]: r = self.visible(self.file_item(a), why=a) self.turn("", [("left_click", {"coordinate": self.at(r), "text": "ctrl"})]) else: first = self.visible(self.file_item(pending[0]), why="the first file") self.turn(self.say("Select all the image files with ctrl+a and upload them together.", "Upload everything at once: select all."), [("left_click", {"coordinate": self.at(first)}), ("key", {"keys": "ctrl+a"})]) self.click('.fc [data-fc="open"]', self.say("Upload them.", "Open them to upload."), why="the Open button") self.from_uploads(el, what, None, fresh=True) def uncover(self, target, what, win, asset_id) -> bool: """Make sure the file window is not over the drop spot: shrink it or move it aside.""" def geo(): return self.js("""(() => { const r = document.querySelector('.fc').getBoundingClientRect(); return [r.left, r.top, r.right, r.bottom]; })()""") tx, ty, m = target[0], target[1], 36 x0, t, r, b = geo() if not (x0 - m < tx < r + m and t - m < ty < b + m): return True st = self.style MINW, MINH = 460, 300 edges = [] # (handle, new edge position): pull one edge back past the drop spot if tx - m - x0 >= MINW: edges.append(("e", [tx - m, (t + b) / 2], "right")) if r - (tx + m) >= MINW: edges.append(("w", [tx + m, (t + b) / 2], "left")) if ty - m - t >= MINH: edges.append(("s", [(x0 + r) / 2, ty - m], "bottom")) if edges and st.chance(st.resize_window): handle, to, side = st.rng.choice(edges) h = self.locate(f".fc .fc-rz-{handle}")[0] start = self.at(h) end = [round(to[0]), start[1]] if handle in "ew" else [start[0], round(to[1])] self.turn(self.say(f"The {win} covers where {what} goes; drag its {side} edge in to make it smaller.", f"Shrink the {win} from its {side} edge so the drop spot is clear."), [("left_click_drag", {"start_coordinate": start, "coordinate": end})]) else: grab = self.js("""(() => { for (const bar of document.querySelectorAll('.fc .fc-drag')) { const r = bar.getBoundingClientRect(), y = r.top + Math.min(r.height / 2, 18); for (let x = r.left + r.width * 0.35; x < r.right - 8; x += 12) { const at = document.elementFromPoint(x, y); if (at && bar.contains(at) && !at.closest('button, input, select')) return [x, y]; } } return null; })()""") if not grab: return False item = self.locate(self.file_item(asset_id)) ix, iy = (item[0]["x"], item[0]["y"]) if item else (x0 + 260, t + 140) # Move right, left, down or up just past the spot, keeping the title bar and the files on screen. moves = [(tx + m - x0, 0), (tx - m - r, 0), (0, ty + m - t), (0, ty - m - b)] ok = [(dx, dy) for dx, dy in moves if 8 < grab[0] + dx < VIEW_W - 8 and 8 < grab[1] + dy < VIEW_H - 8 and 30 < ix + dx < VIEW_W - 30 and 30 < iy + dy < VIEW_H - 30 and t + dy >= 0] if not ok: return False dx, dy = min(ok, key=lambda d: abs(d[0]) + abs(d[1])) self.turn(self.say(f"The {win} is over the spot where {what} goes; drag it aside by its title bar.", f"Move the {win} out of the way first."), [("left_click_drag", {"start_coordinate": [round(grab[0]), round(grab[1])], "coordinate": [round(grab[0] + dx), round(grab[1] + dy)]})]) x0, t, r, b = geo() return not (x0 - 4 < tx < r + 4 and t - 4 < ty < b + 4) def confirm_import(self): """Vector asks how to import a bitmap; the defaults (embed) are right.""" if self.skin != "inkscape": return self.click(".ik-import .btn-primary", self.say("Keep Embed and click OK.", "The import options are fine; OK."), why="the OK button") def close_file_window(self, win): if not self.chooser_os(): return st = self.style roll = st.rng.random() if st.rng else 0.0 if roll < 0.3: self.turn(self.say(f"Done with the {win}; press escape.", f"Close the {win} with escape."), [("key", {"keys": "escape"})]) if self.chooser_os(): # Whichever close control is on screen (a moved window can hang off an edge); else escape. ctrls = ['.fc [data-fc="close"]', '.fc [data-fc="cancel"]'] if roll > 0.65 else ['.fc [data-fc="cancel"]', '.fc [data-fc="close"]'] seen = [c for c in ctrls if any(r.get("visible", True) and 0 <= r["x"] < VIEW_W and 0 <= r["y"] < VIEW_H for r in self.locate(c))] if seen: self.click(seen[0], self.say(f"Close the {win}.", f"The {win} is no longer needed; close it."), why="the close button") else: self.turn(self.say(f"The {win}'s buttons are off screen; press escape to close it.", f"Escape closes the {win}."), [("key", {"keys": "escape"})]) def click_to_place(self, el, what): """Studio: the opened image follows the pointer until a click (or a dragged box) places it.""" st = self.style if not self.locate(".stage.placing"): return a = self.canvas_point(el.x, el.y) z = self.canvas_point(el.x + el.width, el.y + el.height) c = self.canvas_point(el.x + el.width / 2, el.y + el.height / 2) or self.canvas_point( self.js("window.__app.store.doc.width") / 2, self.js("window.__app.store.doc.height") / 2) if a and z and z[0] - a[0] > 12 and st.chance(st.place_box): self.turn(self.say(f"Drag out {what}'s box from its top-left corner.", f"Draw the image's box where {what} sits."), [("left_click_drag", {"start_coordinate": a, "coordinate": z})]) else: self.turn(self.say(f"Click to place it where {what} goes.", f"Click on the canvas to drop {what} there."), [("left_click", {"coordinate": c})]) def from_uploads(self, el, what, assets, fresh=False): """Template: the file is in Uploads; click it in or drag it into place.""" st = self.style self.open("assets") if not fresh and assets and st.rng and len(self.mistakes) < st.max_mistakes and st.chance(st.mistake / 2): others = [a for a in assets if a != el.asset_id and self.js(f"window.__app.uploaded?.has({json.dumps(a)})")] if others: wrong = st.rng.choice(others) self.click(f'.asset-tile[data-asset="{wrong}"]', self.say(f"Add {what}.", f"Place {what}."), why=f"asset {wrong}", train=False) self.mistakes.append({"kind": "wrong_asset", "added": wrong, "meant": el.asset_id}) self.turn(self.say(f"That added {wrong}, not {el.asset_id}. Undo it.", f"Wrong image: this is {wrong}. Undo and pick {el.asset_id}."), [("key", {"keys": "ctrl+z"})]) tile = self.visible(f'.asset-tile[data-asset="{el.asset_id}"]', why=f"the upload {el.asset_id}") target = self.canvas_point(el.x + el.width / 2, el.y + el.height / 2) if st.chance(st.drag_assets) else None lead = "It's in Uploads now. " if fresh else "" if target: self.turn(lead + self.say(f"Drag {what} onto the canvas where it sits in the reference.", f"Drop {what} roughly into place; I'll fine-tune it after."), [("left_click_drag", {"start_coordinate": self.at(tile), "coordinate": target})]) else: self.turn(lead + self.say(f"Add {what}.", f"Click {el.asset_id} in Uploads to add {what} to the page."), [("left_click", {"coordinate": self.at(tile)})]) def add_text(self, el, assets, doc_fonts): st, what = self.style, self._describe(el, assets) if self.skin == "canva": if not self.locate('.cv-rail-btn[data-panel="text"].on'): self.click('.cv-rail-btn[data-panel="text"]', self.say("Open the Text panel.", "Go to Text."), why="the Text tab") self.click(".text-presets .btn-primary", self.say("Add a text box.", "Start a new text box."), why="the add-text button") # Canva drops the box on the page already in editing mode: just type. self.turn(self.say(f"Type {what}.", f"Replace the placeholder with {what}."), [("type", {"text": el.text}), ("key", {"keys": "escape"})]) else: if self.skin == "powerpoint" and not self.locate('.pp-tabs [data-tab="insert"].on'): self.click('.pp-tabs [data-tab="insert"]', self.say("Text Box is on the Insert tab.", "Go to Insert."), why="the Insert tab") self.click('[data-command="toolText"]', self.say(f"Pick the text tool for {what}.", f"Text tool, to add {what}."), why="the text tool") if self.skin != "canva": spot = (self.canvas_point(el.x + min(el.width, 40) / 2, el.y + el.height / 2) if st.chance(st.place_text) else None) if spot is None: stage = self.locate(".stage")[0] spot = self.at(stage) self.turn(self.say(f"Click on the canvas and type {what}.", f"Place the text box and type {what}."), [("left_click", {"coordinate": spot}), ("type", {"text": el.text}), ("key", {"keys": "escape"})]) if st.chance(st.drag_text): cur = self.js("(() => { const e = window.__app.store.selected()[0]; return e && [e.x + e.width / 2, e.y + e.height / 2]; })()") src = self.canvas_point(*cur) if cur else None dst = self.canvas_point(el.x + el.width / 2, el.y + el.height / 2) if src and dst and abs(src[0] - dst[0]) + abs(src[1] - dst[1]) > 12: self.turn(self.say(f"Drag {what} roughly to where it sits in the reference.", f"Move {what} into position; exact numbers next."), [("left_click_drag", {"start_coordinate": src, "coordinate": dst})]) native = self.skin in TEXT_ROUTES and (self.skin in ALWAYS_NATIVE or st.chance(st.native_text)) route = TEXT_ROUTES[self.skin if native else "panel"] if not native: self.open("inspector") elif route.get("pre"): sel, done = route["pre"] if not self.locate(done): self.click(sel, self.say(f"Go to {STEP_NAME.get(sel, 'the text controls')} for the text controls.", "Switch to the text controls."), why=STEP_NAME.get(sel, "the tab")) # The font comes first, as a person would do it: bold and italic only exist # for faces that have them, so styling before choosing the face can be lost. self.set_font(el.font_family, doc_fonts, route["font"]) style_steps = [lambda: self.set_number("font_size", num(el.font_size), what, selector=route["size"])] style_steps.append(lambda: self.set_text_color(el.color or "#000000", route)) if el.align != "left": style_steps.append(lambda: self.set_align(el.align, route)) if self.skin == "figma": style_steps.append(lambda: self.style_figma(el)) elif self.skin == "inkscape": style_steps.append(lambda: self.style_inkscape(el)) elif self.skin == "slides": style_steps.append(lambda: self.style_slides(el, route)) elif self.skin == "powerpoint": style_steps.append(lambda: self.style_powerpoint(el, route)) else: style_steps.append(lambda: [self.toggle(t, want, route["toggle"]) for t, want in ( ("Bold", el.bold), ("Italic", el.italic), ("Underline", el.underline), ("Uppercase", el.uppercase))]) for step in self._order(style_steps): step() pop = POPOVER_FIELDS.get(self.skin, {}) if pop.get("spacing") and (el.line_height != 1 or el.letter_spacing): self.click(pop["spacing"], self.say("Open Spacing for line and letter spacing.", "Spacing next."), why="the Spacing button") if el.line_height != 1: self.set_field("line_height", num(el.line_height), selector='.popover [data-field="line_height"] input') if el.letter_spacing: # Canva reads letter spacing in thousandths of an em self.set_field("letter_spacing", num(el.letter_spacing / el.font_size * 1000), selector='.popover [data-field="letter_spacing"] input') if self.skin == "inkscape": # line height and letter spacing are in the Text tool's controls if el.line_height != 1: self.set_field("line_height", num(el.line_height), selector='.ik-controls [data-field="line_height"] input') if el.letter_spacing: self.set_field("letter_spacing", num(el.letter_spacing), selector='.ik-controls [data-field="letter_spacing"] input') pop = {"spacing": True} if self.skin == "slides": self.spacing_slides(el) pop = {"spacing": True} if self.skin == "powerpoint": self.spacing_powerpoint(el) pop = {"spacing": True} # Otherwise spacing and geometry live in the properties panel. self.open("inspector") if not pop.get("spacing"): fg = self.skin == "figma" # Figma reads line height in % and letter spacing with its unit if el.line_height != 1: self.set_field("line_height", f"{num(el.line_height * 100)}%" if fg else num(el.line_height)) if el.letter_spacing: self.set_field("letter_spacing", f"{num(el.letter_spacing)}px" if fg else num(el.letter_spacing)) if not self.place(el, what, ("width", "x", "y")): self.set_number("width", num(el.width), what) for name in self._order(["x", "y"]): self.set_number(name, num(getattr(el, name)), what) self.finish_common(el, what) def style_inkscape(self, el): """Inkscape: the style dropdown for bold / italic; Text and Font > Features for capitals and underline.""" cur = self.js("(() => { const e = window.__app.store.selected()[0]; return e && [!!e.bold, !!e.italic, !!e.underline, !!e.uppercase]; })()") if cur is None: return want = [bool(el.bold), bool(el.italic)] if cur[:2] != want: name = {(False, False): "Regular", (False, True): "Italic", (True, False): "Bold", (True, True): "Bold Italic"}[tuple(want)] self.click(".ik-controls .ik-style", self.say(f"Open the font style list for {name}.", f"Style: {name}."), why="the font style list") self.click(".pop-menu .menu-item", self.say(f"Choose {name}.", f"{name}."), text=name, why=name) if [cur[2], cur[3]] != [bool(el.underline), bool(el.uppercase)]: if not self.locate('.ik-docktab.on[data-tab="text"]'): self.click('[data-command="dlg_text"]', self.say("Open Text and Font for capitals and underline.", "Text and Font (shift+ctrl+t) has the text features."), why="Text and Font") self.click('.ik-tf .ik-subtab[data-tab="features"]', self.say("Go to the Features tab.", "Features."), why="the Features tab") if cur[3] != bool(el.uppercase): self.click(f'.ik-tf [data-field="uppercase"][data-value="{"true" if el.uppercase else "false"}"]', self.say("Capitals: all caps." if el.uppercase else "Capitals: normal.", "Set the capitals."), why="Capitals") if cur[2] != bool(el.underline): self.click(f'.ik-tf [data-field="underline"][data-value="{"true" if el.underline else "false"}"]', self.say("Decoration: underline." if el.underline else "Decoration: none.", "Set the decoration."), why="Decoration") self.click(".ik-tf .ik-apply", self.say("Apply.", "Click Apply to use it."), why="the Apply button") def style_slides(self, el, route): """Slides: B / I / U in the toolbar; capitals in Format > Text > Capitalization.""" for t, want in (("Bold", el.bold), ("Italic", el.italic), ("Underline", el.underline)): self.toggle(t, want, route["toggle"]) cur = self.js("!!window.__app.store.selected()[0]?.uppercase") if cur != bool(el.uppercase): name = "UPPERCASE" if el.uppercase else "As typed" self.click(".sl-menu", self.say("Capitals are under Format > Text > Capitalization; open Format.", "Format menu for the capitals."), text="Format", why="the Format menu") self.click(".pop-menu .menu-item.has-sub", self.say("Text.", "Go to Text."), text="Text", why="Text") self.click(".menu-sub .menu-item.has-sub", self.say("Capitalization.", "Then Capitalization."), text="Capitalization", why="Capitalization") self.click(".menu-sub .menu-item", self.say(f"{name}.", f"Choose {name}."), text=name, why=name) def spacing_slides(self, el): """Slides: Line & paragraph spacing; a preset when it is one, else Custom spacing (line and letter spacing).""" presets = {1.0: "Single", 1.15: "1.15", 1.5: "1.5", 2.0: "Double"} preset = presets.get(round(el.line_height, 4)) if el.line_height == 1 and not el.letter_spacing: return self.click(".sl-toolbar .sl-spacing", self.say("Open Line & paragraph spacing.", "Spacing menu next."), why="the line spacing button") if preset and not el.letter_spacing: self.click(".pop-menu .menu-item", self.say(f"{preset} spacing.", f"Choose {preset}."), text=preset, why=preset) return self.click(".pop-menu .menu-item", self.say("Custom spacing.", "Open Custom spacing."), text="Custom spacing", why="Custom spacing") if el.line_height != 1: self.set_field("line_height", num(el.line_height), selector='.sl-spacing-dialog [data-field="line_height"] input') if el.letter_spacing: self.set_field("letter_spacing", num(el.letter_spacing), selector='.sl-spacing-dialog [data-field="letter_spacing"] input') self.click(".sl-spacing-dialog .modal-foot .btn-primary", self.say("Apply.", "Click Apply."), why="Apply") def slides_color(self, hexv): """In an open Slides colour menu: the swatch when the palette has it, else CUSTOM + and the hex.""" sw = f'.sl-pop .slc-sw[data-color="#{hexv.lower()}"]' if self.locate(sw) and self.style.chance(0.8): self.click(sw, self.say(f"#{hexv} is in the palette; click it.", f"Pick the #{hexv} swatch."), why="the swatch") return self.click(".sl-pop .slc-add", self.say("Click + under CUSTOM for a custom colour.", "CUSTOM: the + button."), why="the add custom colour button") self.turn(self.say(f"The Hex field is selected; type #{hexv} and press enter for OK.", f"Hex: #{hexv}, then OK."), [("type", {"text": f"#{hexv}"}), ("key", {"keys": "enter"})]) def style_powerpoint(self, el, route): """PowerPoint: B / I / U on the Home tab; All Caps in the Font dialog (More Font Options > Font...).""" for t, want in (("Bold", el.bold), ("Italic", el.italic), ("Underline", el.underline)): self.toggle(t, want, route["toggle"]) if self.js("!!window.__app.store.selected()[0]?.uppercase") != bool(el.uppercase): self.pp_font_dialog("Font...") self.click('.pp-fontdlg [data-field="uppercase"]', self.say("Tick All Caps." if el.uppercase else "Untick All Caps.", "All Caps."), why="All Caps") self.click(".pp-fontdlg .modal-foot .btn-primary", self.say("OK.", "Click OK."), why="OK") def pp_font_dialog(self, item): self.click(".pp-ribbon .pp-morefont", self.say("Open More Font Options.", "The ... after the font colour."), why="More Font Options") if item != "Font...": self.click(".pop-menu .menu-item.has-sub", self.say("Character Spacing.", "Go to Character Spacing."), text="Character Spacing", why="Character Spacing") self.click(".menu-sub .menu-item", self.say(f"{item}", f"Choose {item}"), text=item, why=item) else: self.click(".pop-menu .menu-item", self.say("Font... opens the Font dialog.", "Open the Font dialog."), text="Font...", why="Font...") def pp_dropdown(self, sel, label, why): self.click(sel, self.say(f"Open {why}.", f"{why[0].upper() + why[1:]} dropdown."), why=why) self.click(".pp-ddmenu .menu-item", self.say(f"{label}.", f"Pick {label}."), text=label, why=label) def spacing_powerpoint(self, el): """PowerPoint: Line Spacing (a preset, or Line Spacing Options > Multiple, At); Character Spacing > More Spacing.""" lh = round(el.line_height, 4) if lh != 1: self.click(".pp-ribbon .pp-linespacing", self.say("Open Line Spacing.", "Line spacing next."), why="Line Spacing") if lh in (1.5, 2.0, 2.5, 3.0) and self.style.chance(0.8): self.click(".pop-menu .menu-item", self.say(f"{lh:.1f}.", f"Choose {lh:.1f}."), text=f"{lh:.1f}", why=f"{lh:.1f}") else: self.click(".pop-menu .menu-item", self.say("Line Spacing Options...", "Open the Paragraph dialog."), text="Line Spacing Options...", why="Line Spacing Options") self.pp_dropdown(".pp-paragraph .pp-para-spacing", "Multiple", "the Line Spacing list") self.set_field("line_height", num(el.line_height), selector='.pp-paragraph [data-field="line_height"] input') self.click(".pp-paragraph .modal-foot .btn-primary", self.say("OK.", "Click OK."), why="OK") if el.letter_spacing: self.pp_font_dialog("More Spacing...") self.pp_dropdown(".pp-fontdlg .pp-fd-spacing", "Expanded" if el.letter_spacing > 0 else "Condensed", "the Spacing list") self.set_field("letter_spacing", num(abs(el.letter_spacing)), selector='.pp-fontdlg [data-field="letter_spacing"] input') self.click(".pp-fontdlg .modal-foot .btn-primary", self.say("OK.", "Click OK."), why="OK") def pp_color(self, hexv): """In an open PowerPoint colour gallery: the swatch when it is there, else More Colors... and the Hex field.""" sw = f'.pp-pop .ppc-sw[data-color="#{hexv.lower()}"]' if self.locate(sw) and self.style.chance(0.8): self.click(sw, self.say(f"#{hexv} is in the gallery; click it.", f"Pick #{hexv}."), why="the swatch") return self.click(".pp-pop .pp-morecolors", self.say("More Colors... for a custom colour.", "Open More Colors."), why="More Colors") self.turn(self.say(f"The Hex field is selected; type {hexv.upper()} and press enter for OK.", f"Hex: {hexv.upper()}, then OK."), [("type", {"text": hexv.upper()}), ("key", {"keys": "enter"})]) def pp_pane(self, top=None, sub=None): """Format pane tabs: Shape Options / Text Options, then the icon tab (Size & Properties, Picture, ...).""" if top and self.locate(".pp-format .pp-ftop") and not self.locate(f'.pp-ftop-tab.on[data-top="{top}"]'): name = {"shape": "Shape Options", "text": "Text Options"}[top] self.click(f'.pp-ftop-tab[data-top="{top}"]', self.say(f"{name}.", f"Switch to {name}."), why=name) if sub and self.locate(f'.pp-ftab[data-sub="{sub}"]') and not self.locate(f'.pp-ftab.on[data-sub="{sub}"]'): name = {"size": "Size & Properties", "picture": "Picture", "fill": "Fill & Line"}[sub] self.click(f'.pp-ftab[data-sub="{sub}"]', self.say(f"The {name} tab.", f"Go to {name}."), why=name) def style_figma(self, el): """Figma: Regular / Italic / Bold / Bold Italic from the weight dropdown; underline and case in Type settings.""" cur = self.js("(() => { const e = window.__app.store.selected()[0]; return e && [!!e.bold, !!e.italic, !!e.underline, !!e.uppercase]; })()") if cur is None: return want = [bool(el.bold), bool(el.italic)] if cur[:2] != want: name = {(False, False): "Regular", (False, True): "Italic", (True, False): "Bold", (True, True): "Bold Italic"}[tuple(want)] self.click(".inspector .fg-weight", self.say(f"Open the style dropdown to pick {name}.", f"It is set in {name}; open the weight menu."), why="the font style dropdown") self.click(".pop-menu .menu-item", self.say(f"Choose {name}.", f"{name}."), text=name, why=name) if [cur[2], cur[3]] != [bool(el.underline), bool(el.uppercase)]: self.click(".inspector .fg-typeset-btn", self.say("Open Type settings for decoration and case.", "Type settings next."), why="the Type settings button") if cur[2] != bool(el.underline): v = "underline" if el.underline else "none" self.click(f'.fg-typeset [data-field="decoration"] button[data-value="{v}"]', self.say(f"Decoration: {'underline' if el.underline else 'none'}.", "Set the decoration."), why="Decoration") if cur[3] != bool(el.uppercase): v = "upper" if el.uppercase else "none" self.click(f'.fg-typeset [data-field="case"] button[data-value="{v}"]', self.say(f"Case: {'uppercase' if el.uppercase else 'as typed'}.", "Set the case."), why="Case") self.turn(self.say("Close Type settings.", "Done there; escape."), [("key", {"keys": "escape"})]) def set_text_color(self, color, route): kind, sel = route["color"] hexv = color.lstrip("#")[:6] if kind == "hex": return self.set_field("color", hexv, kind="hex", selector=sel) if kind == "ikfill": # Inkscape: Fill and Stroke > Fill, Flat color, the RGBA entry if not self.locate('.ik-docktab.on[data-tab="fill"]'): self.click(sel, self.say(f"Open Fill and Stroke to set the fill to #{hexv}.", "Fill and Stroke (shift+ctrl+f) for the colour."), why="Fill and Stroke") if not self.locate('.ik-fs .ik-subtab.on[data-tab="fill"]'): self.click('.ik-fs .ik-subtab[data-tab="fill"]', self.say("Go to the Fill tab.", "Fill."), why="the Fill tab") r = self.visible(".ik-fs .ik-rgba", why="the RGBA entry") self.turn(self.say(f"Type {hexv}ff into RGBA and press enter.", f"RGBA: {hexv}ff."), [("triple_click", {"coordinate": self.at(r)}), ("type", {"text": f"{hexv}ff"}), ("key", {"keys": "enter"})]) return if kind == "ppgallery": # PowerPoint: the Font Color dropdown's gallery self.click(sel, self.say(f"Open the Font Color gallery for #{hexv}.", f"Font colour: #{hexv}."), why="the Font Color arrow") return self.pp_color(hexv) if kind == "slmenu": # Slides: the text colour menu self.click(sel, self.say(f"Open the text colour menu to set #{hexv}.", f"Text colour: #{hexv}."), why="the text colour button") return self.slides_color(hexv) if kind == "panel": # Canva: Text color opens in the side panel; search the hex or use "+" if not self.locate('.cv-editpanel[data-panel="color"]'): self.click(sel, self.say(f"Open Text color to set #{hexv}.", f"Colour next: #{hexv}; open the text colour panel."), why="the text colour button") st = self.style if st.chance(0.5): r = self.visible(".cv-colorpanel .cv-colsearch", why="the colour search") self.turn(self.say(f"Type #{hexv} into the colour search and press enter.", f"Search #{hexv}."), [("left_click", {"coordinate": self.at(r)}), ("type", {"text": f"#{hexv}"}), ("key", {"keys": "enter"})]) else: self.click(".cv-colorpanel .cv-addcolor", self.say("Add a new colour with the rainbow +.", "Click + for a custom colour."), why="the add colour button") self.turn(self.say(f"The hex field is selected; type #{hexv}.", f"Enter #{hexv}."), [("type", {"text": f"#{hexv}"}), ("key", {"keys": "enter"}), ("key", {"keys": "escape"})]) return self.click(sel, self.say(f"Open the text colour picker to set #{hexv}.", f"Colour: open the picker for #{hexv}."), why="the text colour button") r = self.visible(".popover .cp-hex", why="the hex field") # Photoshop's Color Picker takes Enter as OK and closes; the other pickers stay open until escape. close = [] if self.skin == "photoshop" else [("key", {"keys": "escape"})] self.turn(self.say(f"Enter #{hexv} as the hex value." + (" Enter is OK." if not close else ""), f"Type #{hexv} into Hex."), [("triple_click", {"coordinate": self.at(r)}), ("type", {"text": f"#{hexv}"}), ("key", {"keys": "enter"}), *close]) def set_align(self, align, route): kind, sel = route["align"] if kind == "ikmenu": # Inkscape: the alignment dropdown label = {"left": "Align left", "center": "Align center", "right": "Align right"}[align] self.click(sel, self.say(f"Open the alignment list to {label.lower()}.", "Alignment dropdown."), why="the alignment dropdown") self.click(".pop-menu .menu-item", self.say(f"{label}.", f"Choose {label.lower()}."), text=label, why=label) return if kind == "ppmenu": # PowerPoint: the Align dropdown label = {"left": "Align Left", "center": "Center", "right": "Align Right"}[align] self.click(sel, self.say(f"Open Align to {label.lower()}.", "Align dropdown."), why="the Align button") self.click(".pp-alignmenu .menu-item", self.say(f"{label}.", f"Choose {label}."), text=label, why=label) return if kind == "slpop": # Slides: the align dropdown's buttons self.click(sel, self.say(f"Open Align to align it {align}.", "Alignment dropdown."), why="the Align button") self.click(f'.sl-pop [data-value="{align}"]', self.say(f"Align {align}.", f"{align.capitalize()}."), why=f"align {align}") return if kind == "cycle": # one button that steps left -> center -> right order = ["left", "center", "right"] cur = self.js("window.__app.store.selected()[0]?.align || 'left'") for _ in range((order.index(align) - order.index(cur)) % 3): self.click(sel, self.say(f"Click alignment until it reads {align}.", f"Step the alignment to {align}."), why="the alignment button") return self.click(sel.format(v=align), self.say(f"Align it {align}.", f"Set alignment to {align}."), why="the alignment buttons") def set_font(self, family, doc_fonts, button=".inspector .font-btn"): st = self.style self.ensure_for(button) self.click(button, self.say(f"Open the font menu to choose {family}.", f"The font should be {family}; open the font picker."), why="the font menu") row = f'.font-pop .font-row[data-family="{family}"]' hits = self.locate(row) if self.skin == "slides": # Slides' menu lists a few fonts; the rest are in More fonts # A font already in the menu (the theme's, Recent, or one added before) is picked there; clicking # it in More fonts would untick it from My fonts instead, as in Slides. if hits: return self.click(row, self.say(f"{family} is in the menu; pick it.", f"Click {family}."), why=f"{family} in the menu") self.click(".sl-fontmenu .menu-item", self.say(f"Open More fonts to find {family}.", "More fonts."), text="More fonts", why="More fonts") self.turn(self.say(f"Search for {family}.", f"Type {family} in the search."), [("type", {"text": family})]) self.click(f'.sl-fonts .font-row[data-family="{family}"]', self.say(f"Tick {family}.", f"Check {family}."), why=f"{family}") return self.click(".sl-fonts .modal-foot .btn-primary", self.say("Done.", "Click Done."), why="Done") in_doc = family in doc_fonts and hits and hits[0].get("visible", True) if not (in_doc and st.chance(st.pick_doc_font)): self.turn(self.say(f"Search for {family}.", f"Type {family} to filter the list."), [("type", {"text": family})]) self.click(row, self.say(f"Pick {family}.", f"Click {family} in the list."), why=f"{family} in the list") def finish_common(self, el, what): if el.angle and not (self.style.chance(self.style.drag_rotate) and self.rotate_by_drag(el, what)): if self.skin == "inkscape": self.rotate_inkscape(el, what) else: self.open("inspector") self.set_field("angle", num(el.angle), thought=self.say( f"It is rotated; set rotation to {num(el.angle)}.", f"Rotate it to {num(el.angle)} degrees.")) if el.opacity < 1: pop = POPOVER_FIELDS.get(self.skin, {}) if pop.get("opacity"): self.click(pop["opacity"], self.say("Open Transparency.", "Set the transparency."), why="the Transparency button") self.set_field("opacity", num(round(el.opacity * 100)), kind="slider", selector='.popover [data-field="opacity"] .range-num') elif self.skin == "inkscape": # the status bar's O: spin self.set_field("opacity", num(round(el.opacity * 100)), selector='.ik-status [data-field="opacity"] input') elif self.skin == "powerpoint": # Format pane: Text Options > Text Fill, or Picture > Transparency self.open("inspector") self.pp_pane(top="text") if el.kind == "text" else self.pp_pane(sub="picture") self.set_field("opacity", num(100 - round(el.opacity * 100)), kind="slider", selector='.pp-format [data-field="opacity"] .range-num', thought=self.say(f"Transparency {num(100 - round(el.opacity * 100))}%.", "Set the transparency.")) elif self.skin == "slides": # Format options > Adjustments: Transparency, 0% is opaque self.open("inspector") if not self.locate('.sl-fo-sec.open[data-sec="adjust"]'): self.click('.sl-fo-sec[data-sec="adjust"] .sl-fo-head', self.say("Expand Adjustments for Transparency.", "Open Adjustments."), why="Adjustments") self.set_field("opacity", num(100 - round(el.opacity * 100)), kind="slider", selector='.sl-format [data-field="opacity"] .range-num', thought=self.say(f"Transparency {num(100 - round(el.opacity * 100))}%.", "Set the transparency.")) elif self.skin == "figma": # a typed "50%" field self.open("inspector") self.set_field("opacity", f"{num(round(el.opacity * 100))}%", selector='.inspector [data-field="opacity"] input') else: self.open("inspector") self.set_field("opacity", num(round(el.opacity * 100)), kind="slider") self.deselect() # ------------------------------------------------------------ direct manipulation def selected_box(self): return self.js("""(() => { const e = window.__app.store.selected()[0]; return e ? {x: e.x, y: e.y, width: e.width, height: e.height, angle: e.angle || 0} : null; })()""") def rotate_inkscape(self, el, what): """Inkscape: Object > Transform, Rotate tab, the angle to turn by (relative), clockwise, Apply.""" cur = self.selected_box()["angle"] delta = el.angle - cur if abs(delta) < 1e-6: return if not self.locate('.ik-docktab.on[data-tab="transform"]'): self.click('[data-command="dlg_transform"]', self.say(f"Open Transform to rotate {what}.", "Transform (shift+ctrl+m)."), why="Transform") self.click('.ik-tr .ik-subtab[data-tab="rotate"]', self.say("Rotate tab.", "Go to Rotate."), why="the Rotate tab") direction = "cw" if delta > 0 else "ccw" if not self.locate(f'.ik-tr .ik-tr-dir button.on[data-value="{direction}"]'): self.click(f'.ik-tr .ik-tr-dir button[data-value="{direction}"]', self.say( "Rotate clockwise." if direction == "cw" else "Rotate counter-clockwise.", "Pick the direction."), why="the direction") self.set_field("rotate_angle", num(abs(delta)), selector='.ik-tr [data-field="rotate_angle"] input', thought=self.say(f"It turns by {num(abs(delta))} degrees.", f"Angle: {num(abs(delta))}.")) self.click(".ik-tr .ik-apply", self.say("Apply the rotation.", "Apply."), why="the Apply button") def ensure_handles(self, what): """Photoshop shows handles only with Show Transform Controls, or in Free Transform (ctrl+t).""" if self.skin != "photoshop" or self.locate(".overlay .sel-box .handle"): return st = self.style box = [r for r in self.locate(".ps-options .ps-transform") if r.get("visible", True)] # only in the Move tool's options if not box or st.chance(0.5): self.turn(self.say(f"Free Transform (ctrl+t) to get handles on {what}.", "ctrl+t for the transform handles."), [("key", {"keys": "ctrl+t"})]) self._free_transform = True else: self.click(".ps-options .ps-transform", self.say("Turn on Show Transform Controls to get the handles.", "Tick Show Transform Controls in the options bar."), why="Show Transform Controls") def end_transform(self): if getattr(self, "_free_transform", False): self._free_transform = False self.turn(self.say("Press enter to commit the transform.", "Commit the transform."), [("key", {"keys": "enter"})]) def handle_at(self, name): hits = self.locate(f".overlay .sel-box .handle.h-{name}") return self.at(hits[0]) if hits and hits[0].get("visible", True) else None def place(self, el, what, fields) -> bool: """Size and position by direct manipulation: handle drag, body drag, arrow-key nudges. Returns False (nothing done) when the plain agent is running, the strategy roll says "type the numbers", or the gesture cannot be made on screen. """ st = self.style roll = st.rng.random() if st.rng else 1.0 if roll >= st.drag_only + st.drag_refine: return False refine = roll >= st.drag_only cur = self.selected_box() if not cur or cur["angle"]: return False # 1. size, with a corner handle (images) or the right-hand side handle (text width) if (el.kind == "image" and (abs(cur["width"] - el.width) > 0.5 or abs(cur["height"] - el.height) > 0.5)) or ( el.kind == "text" and abs(cur["width"] - el.width) > 0.5): self.ensure_handles(what) if el.kind == "image" and (abs(cur["width"] - el.width) > 0.5 or abs(cur["height"] - el.height) > 0.5): h, dst = self.handle_at("se"), self.canvas_point(cur["x"] + el.width, cur["y"] + el.height) if not (h and dst): return False free = abs(cur["width"] / cur["height"] - el.width / el.height) > 0.01 self.turn(self.say(f"Drag the corner handle to size {what} like the reference.", f"Resize {what} from its bottom-right corner." + (" Hold shift to change its proportions." if free else "")), [("left_click_drag", {"start_coordinate": h, "coordinate": dst, **({"text": "shift"} if free else {})})]) elif el.kind == "text" and abs(cur["width"] - el.width) > 0.5: h, dst = self.handle_at("e"), self.canvas_point(cur["x"] + el.width, cur["y"] + cur["height"] / 2) if not (h and dst): return False self.turn(self.say(f"Drag the right edge to make the {what[4:] if what.startswith('the ') else what} box wider.", f"Set the width of {what} by dragging its side handle."), [("left_click_drag", {"start_coordinate": h, "coordinate": [dst[0], h[1]]})]) self.end_transform() # 2. place, dragging the body from its centre to where the reference has it cur = self.selected_box() src = self.canvas_point(cur["x"] + cur["width"] / 2, cur["y"] + cur["height"] / 2) dst = self.canvas_point(el.x + cur["width"] / 2, el.y + cur["height"] / 2) if src and dst and abs(src[0] - dst[0]) + abs(src[1] - dst[1]) > 2: self.turn(self.say(f"Drag {what} into place.", f"Move {what} to where it sits in the reference."), [("left_click_drag", {"start_coordinate": src, "coordinate": dst})]) elif not (src and dst): return False # 3. nudge the last few pixels with the arrow keys (shift = 10 px) cur = self.selected_box() presses, words = [], [] for d, neg, pos in ((el.x - cur["x"], "left", "right"), (el.y - cur["y"], "up", "down")): n = round(abs(d)) tens, ones = divmod(n, 10) key = pos if d > 0 else neg presses += [f"shift+{key}"] * tens + [key] * ones if n: words.append(f"{n} px {key}") if 0 < len(presses) <= 14: self.turn(self.say(f"Nudge it {' and '.join(words)} with the arrow keys.", f"Almost there: {' and '.join(words)}."), [("key", {"keys": presses})]) elif len(presses) > 14: refine = True # 4. anything still visibly off gets typed, as a designer would in the panel cur = self.selected_box() # Drag-only keeps what a careful designer would accept by eye; small elements get # a proportionally tighter tolerance, since a few pixels is a lot of a small box. size = min(el.width, el.height) tol = 0.5 if refine else min(2.5, max(0.75, 0.01 * size)) off = [f for f in fields if abs(cur[f] - getattr(el, f)) > tol] if off: self.open("inspector") label = ", ".join(FIELD_NAME.get(f, f) for f in off) self._pending_thought = self.say(f"The panel shows {label} still off; type the exact value.", f"Fine-tune {label} in the properties panel.") for f in off: self.set_number(f, num(getattr(el, f)), what) return True def rotate_by_drag(self, el, what) -> bool: """Turn the selected element with the rotate handle (Template) or a corner rotate zone.""" import math cur = self.selected_box() if not cur or abs(el.angle) < 4: # the editor snaps to 0 within 3 degrees return False c = self.canvas_point(cur["x"] + cur["width"] / 2, cur["y"] + cur["height"] / 2) self.ensure_handles(what) if self.skin == "inkscape" and c and not self.locate(".overlay .sel-box .rot-zone"): self.turn(self.say(f"Click {what} again for the rotate arrows.", "Click the selection to switch to rotate handles."), [("left_click", {"coordinate": c})]) hits = self.locate(".overlay .sel-box .rot-handle") or self.locate(".overlay .sel-box .rot-zone.z-ne") if not (c and hits): return False hx, hy = hits[0]["x"], hits[0]["y"] t = math.radians(el.angle - cur["angle"]) vx, vy = hx - c[0], hy - c[1] dst = [round(c[0] + vx * math.cos(t) - vy * math.sin(t)), round(c[1] + vx * math.sin(t) + vy * math.cos(t))] if not (0 <= dst[0] < VIEW_W and 0 <= dst[1] < VIEW_H): return False self.turn(self.say(f"Rotate {what} by dragging the rotate handle.", f"Turn {what} to match the reference."), [("left_click_drag", {"start_coordinate": [round(hx), round(hy)], "coordinate": dst})]) self.end_transform() got = self.selected_box()["angle"] if abs(got - el.angle) > 0.3: self.open("inspector") self._pending_thought = self.say(f"Rotation reads {num(got)}; set it exactly.", "Fine-tune the angle.") self.set_field("angle", num(el.angle)) return True def _order(self, items): items = list(items) if self.style.rng and self.style.shuffle_props: self.style.rng.shuffle(items) return items def plan(self, gt, assets) -> str: names = [self._describe(e, assets) for e in gt.elements] listing = "; ".join(f"{i + 1}. {n}" for i, n in enumerate(names)) return self.say( f"The reference has {len(names)} layers, back to front: {listing}. I'll add them in that order " "and type each one's exact position, size and style into the properties panel.", f"Layers from the back: {listing}. Plan: place each layer, then set its numbers exactly.") def build(self, gt, assets: dict | None = None): assets = assets or {} doc_fonts = {e.font_family for e in gt.elements if e.kind == "text"} if self.rec is not None: self._pending_thought = self.plan(gt, assets) if gt.background.lower() != "#ffffff": hexv = gt.background.lstrip("#")[:6] if self.skin == "powerpoint": # Design > Format Background, Fill Color if not self.locate('.pp-tabs [data-tab="design"].on'): self.click('.pp-tabs [data-tab="design"]', self.say("Format Background is on the Design tab.", "Go to Design."), why="the Design tab") if not self.locate('.pp-pane[data-panel="format"]'): self.click('.pp-ribbon [data-open="background"]', self.say("Open Format Background.", "Format Background."), why="Format Background") self.click('.pp-format [data-field="background"]', self.say(f"Open the fill Color for #{hexv}.", "Background colour."), why="the Color button") self.pp_color(hexv) elif self.skin in BG_ROUTES: # e.g. Canva: the Background panel in the side rail opener, done, field = BG_ROUTES[self.skin] if not self.locate(done): self.click(opener, self.say("Open Background to set the page colour.", "Background panel first.") if self.skin == "canva" else self.say("Open Background to set the slide colour.", "The Background button in the toolbar.") if self.skin == "slides" else self.say("Open Document Properties for the page's background colour.", "Document Properties (shift+ctrl+d) has the background.") if self.skin == "inkscape" else self.say("Select Frame 1 to set the background: it is the frame's fill.", "The background is the frame's Fill; click Frame 1 in the layers."), why=BG_NAME[self.skin]) if self.skin == "inkscape": self.click(field, self.say(f"Click the Background color swatch to set #{hexv}.", "Open the background colour."), why="the Background color swatch") self.turn(self.say(f"Type {hexv}ff into RGBA and press enter.", f"RGBA: {hexv}ff."), [("key", {"keys": "ctrl+a"}), ("type", {"text": f"{hexv}ff"}), ("key", {"keys": "enter"}), ("key", {"keys": "escape"})]) elif self.skin == "slides": self.click(field, self.say(f"Open the Color menu for #{hexv}.", "Background colour menu."), why="the Color menu") self.slides_color(hexv) self.click(".modal.sl-bg .modal-foot .btn-primary", self.say("Done.", "Close with Done."), why="Done") elif self.skin == "canva": # the palette button opens Background color in the side panel: search the hex self.click(field, self.say(f"Open the background colour panel for #{hexv}.", "Click the palette for a custom colour."), why="the palette button") r = self.visible(".cv-colorpanel .cv-colsearch", why="the colour search") self.turn(self.say(f"Search #{hexv} and press enter.", f"Type #{hexv}."), [("left_click", {"coordinate": self.at(r)}), ("type", {"text": f"#{hexv}"}), ("key", {"keys": "enter"})]) else: self.set_field("page_background", hexv, kind="hex", selector=field, thought=self.say(f"Set the background colour to #{hexv}.", f"Background: #{hexv}.")) else: self.open("inspector") self.set_field("background", hexv, kind="hex", thought=self.say(f"Set the background colour to {gt.background}.", f"Background first: {gt.background}.")) self.deselect() st = self.style for el in gt.elements: if self.rec is not None and self.mode == "reference" and st.chance(st.view_reference): self.turn(self.say(f"Look at the reference again for {self._describe(el, assets)}.", f"Check the target: where does {self._describe(el, assets)} go?"), [("view_reference", {})]) if el.kind == "image": self.add_image(el, assets) else: self.add_text(el, assets, doc_fonts) self.deselect(self.say("Deselect and compare the canvas with the reference.", "Everything is placed; check it against the reference.")) if self.rec is not None and self.mode == "reference" and st.chance(2 * st.view_reference): self.turn(self.say("Compare with the reference one last time.", "Final check against the target."), [("view_reference", {})]) if self.skin == "inkscape" and not self.locate('.ik-exportdlg [data-command="submit"]'): self.click('[data-command="dlg_export"]', self.say("It matches the reference. Open Export.", "Done; File > Export (shift+ctrl+e)."), why="Export") self.click('[data-command="submit"]', self.say("It matches the reference. Export it.", "The design matches; export it.", "Done. Click Export."), why="the Export button") return self.click(".modal-foot .btn-primary", self.say("Export as PNG.", "Confirm the export."), why="the Export confirm button") def _typo(value: str, rng: random.Random) -> str: digits = [i for i, c in enumerate(value) if c.isdigit()] if len(digits) >= 2 and rng.random() < 0.5: i, j = digits[0], digits[1] s = list(value) s[i], s[j] = s[j], s[i] if "".join(s) != value: return "".join(s) i = rng.choice(digits) return value[:i] + str((int(value[i]) + rng.randrange(1, 9)) % 10) + value[i + 1:] def _result_text(out) -> str: obs = out.observation if obs.error is not None: return f"Error: {obs.error.message}" res = obs.result or {} text = "".join(c.get("text", "") for c in res.get("content", []) if c.get("type") == "text") return text or str(res.get("data", "")) # ================================================================== recording def look_for(task_id: str, seed: int, slot: int | None = None) -> dict: """The editor look for one trajectory. With `slot` (the trajectory's position in a batch) the 12 looks (6 layouts x 2 themes) are dealt out in turn, so any batch is balanced across layouts; the file window's look (macOS, Windows, Linux) cycles alongside. """ rng = random.Random(int(hashlib.sha256(f"{seed}:{task_id}:look".encode()).hexdigest()[:12], 16)) if slot is None: look = {"skin": rng.choice(SKINS), "theme": rng.choice(THEMES), "layout": rng.choice(LAYOUTS)} return {**look, "file_dialog": rng.choice(FILE_DIALOGS)} looks = [(s, t, lay) for lay in LAYOUTS for t in THEMES for s in SKINS] # skin varies fastest skin, theme, layout = looks[slot % len(looks)] # Offset by the look cycle so each layout meets every file window across a batch. dialog = FILE_DIALOGS[(slot + slot // len(looks)) % len(FILE_DIALOGS)] return {"skin": skin, "theme": theme, "layout": layout, "file_dialog": dialog} def record(server: str, pack, task: dict, out_dir: Path, *, mode: str = "reference", seed: int = 0, look: dict | None = None, image_format: str = "webp", natural: bool = True, min_reward: float = 0.98) -> dict: """Record one verified trajectory. Raises if it scores below `min_reward`. Dragging leaves up to a few canvas pixels of residual where the recorder chose not to type the exact number, so a natural trajectory scores 0.98-1.0, not 1.0. """ from openenv.core.mcp_client import MCPToolClient look = look or look_for(task["task_id"], seed) stem = f"{task['task_id']}__{mode}__{look['skin']}-{look['theme']}-{look['layout']}__s{seed}" rng = random.Random(int(hashlib.sha256(f"{seed}:{task['task_id']}:{mode}".encode()).hexdigest()[:12], 16)) rec = Recording(out_dir, stem, image_format) gt = pack.gt_doc(task) assets = {a["id"]: a for a in task["assets"]} t0 = time.time() with MCPToolClient(base_url=server, message_timeout_s=180).sync() as env: r = env.reset(task_id=task["task_id"], surface="gui", mode=mode, max_steps=5000, seed=seed, **look) meta = r.observation.metadata tools = [t for t in env.list_tools(use_cache=False) if t.name != "start_task"] agent = GuiAgent(server, env, meta["episode_id"], look["skin"], style=Style(rng) if natural else Style(), recording=rec, mode=mode) user = [] if mode == "reference": user.append({"type": "image", "image": rec.save_image( base64.b64decode(meta["reference_jpeg_b64"]), "target")}) user.append({"type": "image", "image": rec.save_image(agent.screenshot(), "t000")}) intro = {"reference": "This is the target design. Recreate it.", "description": "Build the design from the brief in your instructions."}[mode] user.append({"type": "text", "text": intro + (" The second image is the editor as it is now." if mode == "reference" else " The image is the editor as it is now.")}) rec.messages[:0] = [{"role": "system", "content": meta["instructions"]}, {"role": "user", "content": user}] final = agent.build(gt, assets) grade = final.observation.metadata.get("grade") if final.done else None if grade is None: out = env.step(CallToolAction(tool_name="submit", arguments={})) grade = out.observation.metadata.get("grade") reward = grade["total"] if grade else None if reward is None or reward < min_reward: worst = sorted(grade["elements"], key=lambda e: e["score"])[:3] if grade else [] raise RuntimeError(f"{stem}: scored {reward}; weakest " f"{[(e['gt_id'], round(e['score'], 3), e['note']) for e in worst]}") return { "messages": rec.messages, "tools": [{"type": "function", "function": {"name": t.name, "description": t.description, "parameters": t.input_schema}} for t in tools], "images": rec.images, # task ids (train-00020) are positions in a task source; crello_id names the design itself "meta": {"task_id": task["task_id"], "crello_id": task.get("crello_id"), "task_source": "corpus" if type(pack).__name__ == "CorpusPack" else "pack", "surface": "gui", "mode": mode, **look, "seed": seed, "reward": reward, "turns": agent.turns, "actions": agent.actions, "masked_turns": rec.masked, "mistakes": agent.mistakes, "n_elements": len(gt.elements), "seconds": round(time.time() - t0, 1), "prompt_version": meta.get("prompt_version"), "recorder_version": RECORDER_VERSION, "viewport": [VIEW_W, VIEW_H]}, } def start_debug_server() -> str: """An in-process server with the locate hook enabled, on a free port.""" import os import socket import uvicorn os.environ["DESIGN_CANVAS_DEBUG"] = "1" s = socket.socket() s.bind(("127.0.0.1", 0)) port = s.getsockname()[1] s.close() os.environ["PORT"] = str(port) from .server.app import create_server server = uvicorn.Server(uvicorn.Config(create_server(), host="127.0.0.1", port=port, log_level="warning", ws_ping_interval=60, ws_ping_timeout=300)) threading.Thread(target=server.run, daemon=True).start() for _ in range(300): try: urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=1) break except Exception: time.sleep(0.1) return f"http://127.0.0.1:{port}" def export(pack, split: str, out: Path, *, limit: int | None = None, start: int = 0, modes=("reference",), seed: int = 0, workers: int = 2, server: str | None = None, max_elements: int = 12, image_format: str = "webp", log=print) -> dict: """Record trajectories for a split into `out/cua.jsonl` (+ images). Resumable.""" from concurrent.futures import ThreadPoolExecutor out = Path(out) out.mkdir(parents=True, exist_ok=True) server = server or start_debug_server() done = set() jsonl = out / "cua.jsonl" if jsonl.exists(): for line in jsonl.read_text().splitlines(): m = json.loads(line)["meta"] done.add((m["task_id"], m["mode"], m["seed"])) jobs, skipped, slot = [], {}, 0 for i in range(start, pack.count(split)): if limit is not None and slot >= limit: # limit is the dataset size, resumes included break task = pack.at(split, i) why = gui_expressible(pack, task) or ( f"more than {max_elements} elements" if task["stats"]["n_elements"] > max_elements else None) if why: skipped[why] = skipped.get(why, 0) + 1 continue mode = modes[(i // len(SKINS)) % len(modes)] # alternate modes, crossed with the layout cycle if (task["task_id"], mode, seed) not in done: # a resume re-deals the same look to the same slot jobs.append((task, mode, look_for(task["task_id"], seed, slot))) slot += 1 lock = threading.Lock() stats = {"recorded": 0, "failed": 0, "turns": [], "actions": [], "images": 0} def job(item): task, mode, look = item try: traj = record(server, pack, task, out, mode=mode, seed=seed, look=look, image_format=image_format) except Exception as err: with lock: stats["failed"] += 1 with open(out / "failures.jsonl", "a") as fh: fh.write(json.dumps({"task_id": task["task_id"], "mode": mode, "seed": seed, "error": f"{type(err).__name__}: {err}"[:500]}) + "\n") log(f" {task['task_id']} {mode}: FAILED {err}") return with lock: with open(jsonl, "a") as fh: fh.write(json.dumps(traj, ensure_ascii=False) + "\n") m = traj["meta"] stats["recorded"] += 1 stats["turns"].append(m["turns"]) stats["actions"].append(m["actions"]) stats["images"] += len(traj["images"]) log(f" {m['task_id']} {mode} {m['skin']}/{m['theme']}/{m['layout']}: reward {m['reward']:.3f}, " f"{m['turns']} turns, {m['actions']} actions, {len(m['mistakes'])} slips, {m['seconds']}s") log(f"recording {len(jobs)} trajectories with {workers} workers -> {out}") with ThreadPoolExecutor(workers) as ex: list(ex.map(job, jobs)) # Summarise the whole dataset on disk, not just this run (a resume adds to it). metas = [json.loads(line)["meta"] for line in jsonl.read_text().splitlines()] if jsonl.exists() else [] summary = {"split": split, "seed": seed, "recorded": len(metas), "failed_this_run": stats["failed"], "skipped": skipped, "images": sum(1 for _ in (out / "images").rglob("*.*")) if (out / "images").exists() else 0, "mean_turns": round(sum(m["turns"] for m in metas) / max(1, len(metas)), 1), "mean_actions": round(sum(m["actions"] for m in metas) / max(1, len(metas)), 1), "mean_reward": round(sum(m["reward"] for m in metas) / max(1, len(metas)), 4), "min_reward": round(min((m["reward"] for m in metas), default=0.0), 4)} (out / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") return summary