Blame for server/gui.py

"""The computer-use surface: a real browser driving the real editor.

For `surface="gui"` the environment opens the editor for the episode in a
headless Chromium page and the agent acts on it with mouse and keyboard, seeing
only screenshots. Nothing is simulated: a click lands on the same DOM a person
would click, the editor turns it into the same operation an MCP call would
make, and the document is scored like any other.

The browser lives on a private event loop in a background thread. A Playwright
browser is bound to the loop that launched it, and the editor page talks back
to this very server over a WebSocket, so driving it from the server's own loop
would deadlock: the tool would wait on the page, and the page would wait on a
server that is blocked waiting for the tool. Tools therefore `await` futures
from this thread and keep the server loop free.

One browser per process, one context per episode (isolated storage), which is
a few tens of megabytes per session rather than a VM.
"""

from __future__ import annotations

import asyncio
import concurrent.futures
import os
import threading

VIEWPORT = {"width": 1280, "height": 800}

# "ctrl" means the platform's shortcut modifier, so an agent's ctrl+a selects
# all and ctrl+z undoes whether the server runs on Linux or macOS.
_KEY_ALIASES = {
    "ctrl": "ControlOrMeta", "control": "ControlOrMeta", "cmd": "Meta", "command": "Meta", "meta": "Meta",
    "super": "Meta", "win": "Meta", "alt": "Alt", "option": "Alt", "shift": "Shift",
    "enter": "Enter", "return": "Enter", "esc": "Escape", "escape": "Escape",
    "backspace": "Backspace", "delete": "Delete", "del": "Delete", "tab": "Tab",
    "space": " ", "up": "ArrowUp", "down": "ArrowDown", "left": "ArrowLeft",
    "right": "ArrowRight", "home": "Home", "end": "End", "pageup": "PageUp",
    "pagedown": "PageDown", "plus": "+", "minus": "-",
    "arrowup": "ArrowUp", "arrowdown": "ArrowDown", "arrowleft": "ArrowLeft", "arrowright": "ArrowRight",
}


def normalize_keys(combo: str) -> str:
    """'ctrl+shift+z' -> 'Control+Shift+z' (Playwright's spelling)."""
    parts = [p for p in combo.replace(" ", "").split("+") if p] or [combo]
    out = []
    for p in parts:
        low = p.lower()
        if low in _KEY_ALIASES:
            out.append(_KEY_ALIASES[low])
        elif len(p) == 1:
            out.append(p)
        elif low.startswith("f") and low[1:].isdigit():
            out.append(p.upper())
        else:
            out.append(p[:1].upper() + p[1:])
    return "+".join(out)


def self_url() -> str:
    url = os.environ.get("DESIGN_CANVAS_SELF_URL")
    if url:
        return url.rstrip("/")
    return f"http://127.0.0.1:{os.environ.get('PORT', '8000')}"


class GuiDriver:
    _instance: GuiDriver | None = None
    _instance_lock = threading.Lock()

    @classmethod
    def get(cls) -> GuiDriver:
        with cls._instance_lock:
            if cls._instance is None:
                cls._instance = cls()
            return cls._instance

    def __init__(self):
        self._loop = asyncio.new_event_loop()
        self._thread = threading.Thread(target=self._loop.run_forever, name="gui-driver",
                                        daemon=True)
        self._thread.start()
        self._pw = None
        self._browser = None
        self._pages: dict[str, tuple] = {}  # episode_id -> (context, page)
        self._remote: dict[str, object] = {}  # episode_id -> remote screen (desktop agent), instead of a page
        self._start_lock = None

    # ------------------------------------------------------------ plumbing

    def submit(self, coro) -> concurrent.futures.Future:
        return asyncio.run_coroutine_threadsafe(coro, self._loop)

    async def call(self, coro, timeout: float = 30.0):
        """Await `coro` on the driver loop from any other loop."""
        fut = self.submit(coro)
        return await asyncio.wait_for(asyncio.wrap_future(fut), timeout)

    def call_sync(self, coro, timeout: float = 30.0):
        return self.submit(coro).result(timeout)

    async def _browser_ready(self):
        if self._start_lock is None:
            self._start_lock = asyncio.Lock()
        async with self._start_lock:
            if self._browser is None or not self._browser.is_connected():
                try:
                    from playwright.async_api import async_playwright
                except ImportError as exc:  # pragma: no cover - depends on extras
                    raise RuntimeError(
                        "the gui surface needs Playwright: pip install 'openenv-design-canvas[gui]'"
                        " && playwright install chromium"
                    ) from exc
                self._pw = await async_playwright().start()
                self._browser = await self._pw.chromium.launch(
                    args=["--disable-dev-shm-usage", "--font-render-hinting=none",
                          "--disable-lcd-text"]
                )
        return self._browser

    # ------------------------------------------------------------ pages

    async def _open(self, episode_id: str, url: str, cookies=None, ready=None, headers=None):
        await self._close(episode_id)
        browser = await self._browser_ready()
        context = await browser.new_context(viewport=VIEWPORT, device_scale_factor=1, extra_http_headers=headers)
        if headers:
            # the headers carry a sandbox's access token: the page may reach that app and nothing else
            origin = "/".join(url.split("/")[:3])

            async def only_app(route):
                await (route.continue_() if route.request.url.startswith(origin) else route.abort("blockedbyclient"))

            await context.route("**/*", only_app)
        if cookies:
            await context.add_cookies(cookies)
        page = await context.new_page()
        await page.goto(url, wait_until="load")
        if ready:
            await page.wait_for_selector(ready, timeout=120000)
            await page.wait_for_timeout(2500)  # a real app keeps loading fonts and its renderer after first paint
        else:
            await page.wait_for_function("window.designEditor && window.designEditor.ready === true",
                                         timeout=20000, polling=100)
        await self._settle(page)
        self._pages[episode_id] = (context, page)
        return True

    async def _close(self, episode_id: str):
        entry = self._pages.pop(episode_id, None)
        if entry:
            try:
                await entry[0].close()
            except Exception:
                pass

    def _page(self, episode_id: str):
        entry = self._pages.get(episode_id)
        if entry is None:
            raise RuntimeError("no editor page for this episode; call reset(surface='gui')")
        return entry[1]

    async def _settle(self, page, ms: int = 60):
        # Let the editor finish pending operations and repaint before the next
        # screenshot, so the agent never acts on a half-updated frame.
        try:
            await page.evaluate(
                "() => (window.designEditor && window.designEditor.whenIdle)"
                " ? window.designEditor.whenIdle(2000) : null"
            )
        except Exception:
            pass
        await page.wait_for_timeout(ms)

    # ------------------------------------------------------------ actions

    def attach_remote(self, episode_id: str, client) -> None:
        """Drive this episode through a remote screen (a desktop agent: realapps.desktop.client.DesktopClient)."""
        self._remote[episode_id] = client

    def detach_remote(self, episode_id: str) -> None:
        self._remote.pop(episode_id, None)

    def open(self, episode_id: str, url: str, cookies: list[dict] | None = None, ready: str | None = None,
             headers: dict[str, str] | None = None):
        """Open the editor for an episode. A real app (realapps) passes its session cookies and a CSS selector
        that appears once its editor has loaded, instead of the built-in editor's ready flag, and, when it runs in
        a sandbox, the headers that admit the browser (requests to any other origin are then blocked)."""
        return self.submit(self._open(episode_id, url, cookies, ready, headers))

    def close(self, episode_id: str):
        return self.submit(self._close(episode_id))

    async def screenshot(self, episode_id: str) -> bytes:
        if (r := self._remote.get(episode_id)) is not None:
            return await asyncio.to_thread(r.screenshot)
        async def run():
            page = self._page(episode_id)
            await self._settle(page, 30)
            return await page.screenshot(type="png")

        return await self.call(run())

    async def click(self, episode_id, x, y, button="left", clicks=1, modifiers=None):
        if (r := self._remote.get(episode_id)) is not None:
            return await asyncio.to_thread(r.input, type={1: "click", 2: "double", 3: "triple"}.get(clicks, "click") if button != "right" else "right", x=x, y=y, modifiers=[normalize_keys(m) for m in (modifiers or [])])
        async def run():
            page = self._page(episode_id)
            mods = [normalize_keys(m) for m in (modifiers or [])]
            for m in mods:
                await page.keyboard.down(m)
            try:
                await page.mouse.click(x, y, button=button, click_count=clicks, delay=20)
            finally:
                for m in reversed(mods):
                    await page.keyboard.up(m)
            await self._settle(page)

        return await self.call(run())

    async def move(self, episode_id, x, y):
        if (r := self._remote.get(episode_id)) is not None:
            return await asyncio.to_thread(r.input, type="move", x=x, y=y)
        async def run():
            page = self._page(episode_id)
            await page.mouse.move(x, y, steps=4)
            await self._settle(page, 30)

        return await self.call(run())

    async def drag(self, episode_id, sx, sy, ex, ey, modifiers=None):
        if (r := self._remote.get(episode_id)) is not None:
            return await asyncio.to_thread(r.input, type="drag", **{"from": [sx, sy], "to": [ex, ey]})
        async def run():
            page = self._page(episode_id)
            mods = [normalize_keys(m) for m in (modifiers or [])]
            await page.mouse.move(sx, sy)
            for m in mods:
                await page.keyboard.down(m)
            await page.mouse.down()
            steps = max(6, int(((ex - sx) ** 2 + (ey - sy) ** 2) ** 0.5 / 25))
            await page.mouse.move(ex, ey, steps=steps)
            await page.mouse.up()
            for m in reversed(mods):
                await page.keyboard.up(m)
            await self._settle(page, 90)

        return await self.call(run())

    async def scroll(self, episode_id, x, y, dx, dy, modifiers=None):
        if (r := self._remote.get(episode_id)) is not None:
            return await asyncio.to_thread(r.input, type="scroll", x=x, y=y, dx=round(dx / 100), dy=round(dy / 100))
        async def run():
            page = self._page(episode_id)
            await page.mouse.move(x, y)
            mods = [normalize_keys(m) for m in (modifiers or [])]
            for m in mods:
                await page.keyboard.down(m)
            await page.mouse.wheel(dx, dy)
            for m in reversed(mods):
                await page.keyboard.up(m)
            await self._settle(page, 80)

        return await self.call(run())

    async def type_text(self, episode_id, text: str):
        if (r := self._remote.get(episode_id)) is not None:
            return await asyncio.to_thread(r.input, type="type", text=text)
        async def run():
            page = self._page(episode_id)
            await page.keyboard.type(text, delay=8)
            await self._settle(page)

        return await self.call(run(), timeout=60)

    async def key(self, episode_id, combo: str):
        if (r := self._remote.get(episode_id)) is not None:
            return await asyncio.to_thread(r.input, type="key", keys=[normalize_keys(p) for p in combo.split()])
        async def run():
            page = self._page(episode_id)
            for part in combo.split():
                await page.keyboard.press(normalize_keys(part))
            await self._settle(page)

        return await self.call(run())

    async def wait(self, episode_id, seconds: float):
        if episode_id in self._remote:
            return await asyncio.sleep(max(0.0, min(seconds, 10.0)))
        async def run():
            page = self._page(episode_id)
            await page.wait_for_timeout(int(max(0.0, min(seconds, 10.0)) * 1000))

        return await self.call(run(), timeout=15)