File size: 5,231 Bytes
8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 8b9b8c8 92b12f2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | """The end-to-end test: a real display, real capture, real OCR, a real click.
This is the only test that proves the loop drives a *computer* rather than a
synthetic screenshot. It is opt-in because it moves the real mouse and clicks:
TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py
Run it on a throwaway desktop (VM, container with Xvfb), never your working
session. With Xvfb it needs no hardware at all:
Xvfb :99 -screen 0 1280x800x24 &
DISPLAY=:99 TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py
"""
import os
import pathlib
import subprocess
import sys
import time
import pytest
from tiny_computer_agent.act import PyAutoGuiExecutor
from tiny_computer_agent.desktop import display_available
from tiny_computer_agent.observe import GrabScreen
from tiny_computer_agent.perception import (
IndexedScreen, OcrDetector, detector_available,
)
OPT_IN = os.environ.get("TINY_COMPUTER_AGENT_LIVE_TEST") == "1"
pytestmark = pytest.mark.skipif(
not OPT_IN,
reason="moves the real mouse: set TINY_COMPUTER_AGENT_LIVE_TEST=1 on a throwaway desktop",
)
WINDOW = (120, 120)
BUTTON = (20, 155, 110, 40) # x, y, width, height inside the window
READY_TIMEOUT = 20.0
APP = '''import pathlib, tkinter as tk
MARKER = pathlib.Path(r"__MARKER__")
READY = pathlib.Path(str(MARKER) + ".ready")
root = tk.Tk()
root.title("Document")
root.geometry("420x230+120+120")
root.configure(bg="#f5f6f8")
title = tk.Label(root, text="Untitled document - unsaved changes", bg="#f5f6f8")
title.place(x=20, y=18)
tk.Label(root, text="File name", bg="#f5f6f8").place(x=20, y=70)
entry = tk.Entry(root)
entry.place(x=20, y=95, width=300, height=28)
entry.insert(0, "report.txt")
def chosen(what):
MARKER.write_text(what)
title.config(text="Saved to disk")
for widget in buttons:
widget.place_forget()
buttons = [tk.Button(root, text="Save", command=lambda: chosen("Save"))]
buttons[0].place(x=20, y=155, width=110, height=40)
root.update()
READY.write_text("mapped")
root.mainloop()
'''
@pytest.fixture
def save_dialog(tmp_path):
"""A real Tk window with a real Save button, on the real display."""
if not display_available()[0]:
pytest.skip("no display session to drive")
if not detector_available("ocr")[0]:
pytest.skip("this test needs OCR to find the button")
marker = tmp_path / "clicked.txt"
ready = pathlib.Path(str(marker) + ".ready")
app = tmp_path / "app.py"
app.write_text(APP.replace("__MARKER__", str(marker)))
process = subprocess.Popen([sys.executable, str(app)])
try:
deadline = time.time() + READY_TIMEOUT
while time.time() < deadline and not ready.exists():
if process.poll() is not None:
pytest.fail(f"the test window exited immediately (code {process.returncode})")
time.sleep(0.2)
assert ready.exists(), "the test window never mapped"
yield marker
finally:
# Wait for the window to actually disappear: a ghost window left on the
# display makes the *next* test's OCR see two identical buttons.
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired: # pragma: no cover
process.kill()
process.wait(timeout=5)
def _look() -> tuple:
"""(observation, elements) - one real capture, one real OCR pass."""
observation = GrabScreen(monitor=1).capture()
return observation, OcrDetector().detect(observation.frame)
def test_ocr_finds_a_real_button_at_the_right_pixels(save_dialog):
"""Pixels in, a label whose coordinates really are the button's."""
_, elements = _look()
save = [e for e in elements if e.matches("Save")]
assert save, f"OCR found no Save button, only {[e.label for e in elements]}"
x, y = save[0].center
assert WINDOW[0] + BUTTON[0] <= x <= WINDOW[0] + BUTTON[0] + BUTTON[2], f"{x} outside the button"
assert WINDOW[1] + BUTTON[1] <= y <= WINDOW[1] + BUTTON[1] + BUTTON[3], f"{y} outside the button"
def test_a_real_click_lands_on_the_button(save_dialog):
"""The whole loop: see it, choose it, click it, and the app reacts."""
from tiny_computer_agent import AgentLoop, PlannerBrain, Policy
_, elements = _look()
labels = [e.label for e in elements]
save = [e for e in elements if e.matches("Save")]
assert save, f"nothing named Save on screen before clicking: {labels}"
# IndexedScreen, not a bare GrabScreen: capture gives pixels, the detector
# gives labels, and without labels there is nothing for a brain to choose.
episode = AgentLoop(
screen=IndexedScreen(GrabScreen(monitor=1), OcrDetector()),
brain=PlannerBrain(),
executor=PyAutoGuiExecutor(),
policy=Policy(dry_run=False, step_budget=2),
verify=lambda observation: "saved" in observation.text.lower(),
).run("click 'Save' to save the document")
detail = episode.render() + f"\nclicked at {save[0].center}, labels were {labels}"
assert save_dialog.exists(), f"no click landed on the button\n{detail}"
assert save_dialog.read_text() == "Save"
assert episode.success, detail
|