| """The end-to-end test: a real display, real capture, real OCR, a real click. |
| |
| This is the only test that proves the loop drives a *computer* rather than a |
| synthetic screenshot. It is opt-in because it moves the real mouse and clicks: |
| |
| TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py |
| |
| Run it on a throwaway desktop (VM, container with Xvfb), never your working |
| session. With Xvfb it needs no hardware at all: |
| |
| Xvfb :99 -screen 0 1280x800x24 & |
| DISPLAY=:99 TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py |
| """ |
|
|
| import os |
| import pathlib |
| import subprocess |
| import sys |
| import time |
|
|
| import pytest |
|
|
| from tiny_computer_agent.act import PyAutoGuiExecutor |
| from tiny_computer_agent.desktop import display_available |
| from tiny_computer_agent.observe import GrabScreen |
| from tiny_computer_agent.perception import ( |
| IndexedScreen, OcrDetector, detector_available, |
| ) |
|
|
| OPT_IN = os.environ.get("TINY_COMPUTER_AGENT_LIVE_TEST") == "1" |
|
|
| pytestmark = pytest.mark.skipif( |
| not OPT_IN, |
| reason="moves the real mouse: set TINY_COMPUTER_AGENT_LIVE_TEST=1 on a throwaway desktop", |
| ) |
|
|
| WINDOW = (120, 120) |
| BUTTON = (20, 155, 110, 40) |
| READY_TIMEOUT = 20.0 |
|
|
| APP = '''import pathlib, tkinter as tk |
| |
| MARKER = pathlib.Path(r"__MARKER__") |
| READY = pathlib.Path(str(MARKER) + ".ready") |
| |
| root = tk.Tk() |
| root.title("Document") |
| root.geometry("420x230+120+120") |
| root.configure(bg="#f5f6f8") |
| title = tk.Label(root, text="Untitled document - unsaved changes", bg="#f5f6f8") |
| title.place(x=20, y=18) |
| tk.Label(root, text="File name", bg="#f5f6f8").place(x=20, y=70) |
| entry = tk.Entry(root) |
| entry.place(x=20, y=95, width=300, height=28) |
| entry.insert(0, "report.txt") |
| |
| def chosen(what): |
| MARKER.write_text(what) |
| title.config(text="Saved to disk") |
| for widget in buttons: |
| widget.place_forget() |
| |
| buttons = [tk.Button(root, text="Save", command=lambda: chosen("Save"))] |
| buttons[0].place(x=20, y=155, width=110, height=40) |
| root.update() |
| READY.write_text("mapped") |
| root.mainloop() |
| ''' |
|
|
|
|
| @pytest.fixture |
| def save_dialog(tmp_path): |
| """A real Tk window with a real Save button, on the real display.""" |
| if not display_available()[0]: |
| pytest.skip("no display session to drive") |
| if not detector_available("ocr")[0]: |
| pytest.skip("this test needs OCR to find the button") |
|
|
| marker = tmp_path / "clicked.txt" |
| ready = pathlib.Path(str(marker) + ".ready") |
| app = tmp_path / "app.py" |
| app.write_text(APP.replace("__MARKER__", str(marker))) |
|
|
| process = subprocess.Popen([sys.executable, str(app)]) |
| try: |
| deadline = time.time() + READY_TIMEOUT |
| while time.time() < deadline and not ready.exists(): |
| if process.poll() is not None: |
| pytest.fail(f"the test window exited immediately (code {process.returncode})") |
| time.sleep(0.2) |
| assert ready.exists(), "the test window never mapped" |
| yield marker |
| finally: |
| |
| |
| process.terminate() |
| try: |
| process.wait(timeout=10) |
| except subprocess.TimeoutExpired: |
| process.kill() |
| process.wait(timeout=5) |
|
|
|
|
| def _look() -> tuple: |
| """(observation, elements) - one real capture, one real OCR pass.""" |
| observation = GrabScreen(monitor=1).capture() |
| return observation, OcrDetector().detect(observation.frame) |
|
|
|
|
| def test_ocr_finds_a_real_button_at_the_right_pixels(save_dialog): |
| """Pixels in, a label whose coordinates really are the button's.""" |
| _, elements = _look() |
|
|
| save = [e for e in elements if e.matches("Save")] |
| assert save, f"OCR found no Save button, only {[e.label for e in elements]}" |
| x, y = save[0].center |
| assert WINDOW[0] + BUTTON[0] <= x <= WINDOW[0] + BUTTON[0] + BUTTON[2], f"{x} outside the button" |
| assert WINDOW[1] + BUTTON[1] <= y <= WINDOW[1] + BUTTON[1] + BUTTON[3], f"{y} outside the button" |
|
|
|
|
| def test_a_real_click_lands_on_the_button(save_dialog): |
| """The whole loop: see it, choose it, click it, and the app reacts.""" |
| from tiny_computer_agent import AgentLoop, PlannerBrain, Policy |
|
|
| _, elements = _look() |
| labels = [e.label for e in elements] |
| save = [e for e in elements if e.matches("Save")] |
| assert save, f"nothing named Save on screen before clicking: {labels}" |
|
|
| |
| |
| episode = AgentLoop( |
| screen=IndexedScreen(GrabScreen(monitor=1), OcrDetector()), |
| brain=PlannerBrain(), |
| executor=PyAutoGuiExecutor(), |
| policy=Policy(dry_run=False, step_budget=2), |
| verify=lambda observation: "saved" in observation.text.lower(), |
| ).run("click 'Save' to save the document") |
|
|
| detail = episode.render() + f"\nclicked at {save[0].center}, labels were {labels}" |
| assert save_dialog.exists(), f"no click landed on the button\n{detail}" |
| assert save_dialog.read_text() == "Save" |
| assert episode.success, detail |
|
|
