"""The end-to-end test: a real display, real capture, real OCR, a real click. This is the only test that proves the loop drives a *computer* rather than a synthetic screenshot. It is opt-in because it moves the real mouse and clicks: TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py Run it on a throwaway desktop (VM, container with Xvfb), never your working session. With Xvfb it needs no hardware at all: Xvfb :99 -screen 0 1280x800x24 & DISPLAY=:99 TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py """ import os import pathlib import subprocess import sys import time import pytest from tiny_computer_agent.act import PyAutoGuiExecutor from tiny_computer_agent.desktop import display_available from tiny_computer_agent.observe import GrabScreen from tiny_computer_agent.perception import ( IndexedScreen, OcrDetector, detector_available, ) OPT_IN = os.environ.get("TINY_COMPUTER_AGENT_LIVE_TEST") == "1" pytestmark = pytest.mark.skipif( not OPT_IN, reason="moves the real mouse: set TINY_COMPUTER_AGENT_LIVE_TEST=1 on a throwaway desktop", ) WINDOW = (120, 120) BUTTON = (20, 155, 110, 40) # x, y, width, height inside the window READY_TIMEOUT = 20.0 APP = '''import pathlib, tkinter as tk MARKER = pathlib.Path(r"__MARKER__") READY = pathlib.Path(str(MARKER) + ".ready") root = tk.Tk() root.title("Document") root.geometry("420x230+120+120") root.configure(bg="#f5f6f8") title = tk.Label(root, text="Untitled document - unsaved changes", bg="#f5f6f8") title.place(x=20, y=18) tk.Label(root, text="File name", bg="#f5f6f8").place(x=20, y=70) entry = tk.Entry(root) entry.place(x=20, y=95, width=300, height=28) entry.insert(0, "report.txt") def chosen(what): MARKER.write_text(what) title.config(text="Saved to disk") for widget in buttons: widget.place_forget() buttons = [tk.Button(root, text="Save", command=lambda: chosen("Save"))] buttons[0].place(x=20, y=155, width=110, height=40) root.update() READY.write_text("mapped") root.mainloop() ''' @pytest.fixture def save_dialog(tmp_path): """A real Tk window with a real Save button, on the real display.""" if not display_available()[0]: pytest.skip("no display session to drive") if not detector_available("ocr")[0]: pytest.skip("this test needs OCR to find the button") marker = tmp_path / "clicked.txt" ready = pathlib.Path(str(marker) + ".ready") app = tmp_path / "app.py" app.write_text(APP.replace("__MARKER__", str(marker))) process = subprocess.Popen([sys.executable, str(app)]) try: deadline = time.time() + READY_TIMEOUT while time.time() < deadline and not ready.exists(): if process.poll() is not None: pytest.fail(f"the test window exited immediately (code {process.returncode})") time.sleep(0.2) assert ready.exists(), "the test window never mapped" yield marker finally: # Wait for the window to actually disappear: a ghost window left on the # display makes the *next* test's OCR see two identical buttons. process.terminate() try: process.wait(timeout=10) except subprocess.TimeoutExpired: # pragma: no cover process.kill() process.wait(timeout=5) def _look() -> tuple: """(observation, elements) - one real capture, one real OCR pass.""" observation = GrabScreen(monitor=1).capture() return observation, OcrDetector().detect(observation.frame) def test_ocr_finds_a_real_button_at_the_right_pixels(save_dialog): """Pixels in, a label whose coordinates really are the button's.""" _, elements = _look() save = [e for e in elements if e.matches("Save")] assert save, f"OCR found no Save button, only {[e.label for e in elements]}" x, y = save[0].center assert WINDOW[0] + BUTTON[0] <= x <= WINDOW[0] + BUTTON[0] + BUTTON[2], f"{x} outside the button" assert WINDOW[1] + BUTTON[1] <= y <= WINDOW[1] + BUTTON[1] + BUTTON[3], f"{y} outside the button" def test_a_real_click_lands_on_the_button(save_dialog): """The whole loop: see it, choose it, click it, and the app reacts.""" from tiny_computer_agent import AgentLoop, PlannerBrain, Policy _, elements = _look() labels = [e.label for e in elements] save = [e for e in elements if e.matches("Save")] assert save, f"nothing named Save on screen before clicking: {labels}" # IndexedScreen, not a bare GrabScreen: capture gives pixels, the detector # gives labels, and without labels there is nothing for a brain to choose. episode = AgentLoop( screen=IndexedScreen(GrabScreen(monitor=1), OcrDetector()), brain=PlannerBrain(), executor=PyAutoGuiExecutor(), policy=Policy(dry_run=False, step_budget=2), verify=lambda observation: "saved" in observation.text.lower(), ).run("click 'Save' to save the document") detail = episode.render() + f"\nclicked at {save[0].center}, labels were {labels}" assert save_dialog.exists(), f"no click landed on the button\n{detail}" assert save_dialog.read_text() == "Save" assert episode.success, detail