Publish tests/ from marimo

tests/test_act.py

@@ -51,11 +51,16 @@ def test_live_executor_treats_done_as_a_no_op():
 
 
 def test_live_executor_needs_pyautogui_installed():
-    """Real input on a machine without pyautogui fails loudly, not silently."""
+    """Real input on a machine that cannot run pyautogui fails loudly, not silently.
+
+    pyautogui installed is not the same as pyautogui usable: with no display it
+    raises KeyError('DISPLAY') on import, so this must catch Exception rather than
+    ImportError or the failure escapes as a traceback instead of InputUnavailable.
+    """
     try:
         import pyautogui  # noqa: F401
-    except ImportError:
+    except Exception:
         with pytest.raises(InputUnavailable, match="pyautogui"):
             PyAutoGuiExecutor().execute(Action("type", text="hello"))
     else:
-        pytest.skip("pyautogui is installed here")
+        pytest.skip("pyautogui is usable here")

tests/test_live_desktop.py

@@ -0,0 +1,112 @@
+"""The end-to-end test: a real display, real capture, real OCR, a real click.
+
+This is the only test that proves the loop drives a *computer* rather than a
+synthetic screenshot. It is opt-in because it moves the real mouse and clicks:
+
+    TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py
+
+Run it on a throwaway desktop (VM, container with Xvfb), never your working
+session. With Xvfb it needs no hardware:
+
+    Xvfb :99 -screen 0 1280x800x24 &
+    DISPLAY=:99 TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py
+"""
+
+import os
+import pathlib
+import subprocess
+import sys
+import time
+
+import pytest
+
+from tiny_computer_agent.act import PyAutoGuiExecutor
+from tiny_computer_agent.actions import Action
+from tiny_computer_agent.desktop import display_available
+from tiny_computer_agent.observe import GrabScreen
+from tiny_computer_agent.perception import OcrDetector, detector_available
+
+OPT_IN = os.environ.get("TINY_COMPUTER_AGENT_LIVE_TEST") == "1"
+
+pytestmark = pytest.mark.skipif(
+    not OPT_IN,
+    reason="moves the real mouse: set TINY_COMPUTER_AGENT_LIVE_TEST=1 on a throwaway desktop",
+)
+
+APP = '''import pathlib, tkinter as tk
+
+MARKER = pathlib.Path("%(marker)s")
+WINDOW = (120, 120)
+BUTTON = (20, 155, 110, 40)          # x, y, width, height inside the window
+
+root = tk.Tk()
+root.title("Document")
+root.geometry("420x230+%%d+%%d" %% WINDOW)
+root.configure(bg="#f5f6f8")
+title = tk.Label(root, text="Untitled document - unsaved changes", bg="#f5f6f8")
+title.place(x=20, y=18)
+tk.Label(root, text="File name", bg="#f5f6f8").place(x=20, y=70)
+entry = tk.Entry(root)
+entry.place(x=20, y=95, width=300, height=28)
+entry.insert(0, "report.txt")
+
+def chosen(what):
+    MARKER.write_text(what)
+    title.config(text="Saved to disk")
+    for widget in buttons:
+        widget.place_forget()
+
+buttons = [tk.Button(root, text="Save", command=lambda: chosen("Save"))]
+buttons[0].place(x=BUTTON[0], y=BUTTON[1], width=BUTTON[2], height=BUTTON[3])
+root.update()
+root.mainloop()
+'''
+
+
+@pytest.fixture
+def save_dialog(tmp_path):
+    """A real Tk window with a real Save button, on the real display."""
+    if not display_available()[0]:
+        pytest.skip("no display session to drive")
+    if not detector_available("ocr")[0]:
+        pytest.skip("this test needs OCR to find the button")
+    marker = tmp_path / "clicked.txt"
+    app = tmp_path / "app.py"
+    app.write_text(APP % {"marker": marker})
+    process = subprocess.Popen([sys.executable, str(app)])
+    try:
+        time.sleep(2.5)                      # let Tk map the window
+        yield marker, (120, 120), (20, 155, 110, 40)
+    finally:
+        process.terminate()
+
+
+def test_ocr_finds_a_real_button_at_the_right_pixels(save_dialog):
+    """Pixels in, a label whose coordinates really are the button's."""
+    marker, window, (bx, by, bw, bh) = save_dialog
+    observation = GrabScreen(monitor=1).capture()
+    elements = OcrDetector().detect(observation.frame)
+
+    save = [e for e in elements if e.matches("Save")]
+    assert save, f"OCR found no Save button, only {[e.label for e in elements]}"
+    centre = save[0].center
+    assert window[0] + bx <= centre[0] <= window[0] + bx + bw, f"{centre} outside the button"
+    assert window[1] + by <= centre[1] <= window[1] + by + bh, f"{centre} outside the button"
+
+
+def test_a_real_click_lands_on_the_button(save_dialog):
+    """The whole loop: see it, choose it, click it, and the app reacts."""
+    from tiny_computer_agent import AgentLoop, PlannerBrain, Policy
+
+    marker, _, _ = save_dialog
+    episode = AgentLoop(
+        screen=GrabScreen(monitor=1),
+        brain=PlannerBrain(),
+        executor=PyAutoGuiExecutor(),
+        policy=Policy(dry_run=False, step_budget=2),
+        verify=lambda observation: "saved" in observation.text.lower(),
+    ).run("click 'Save' to save the document")
+
+    assert marker.exists(), "no click landed on the button"
+    assert marker.read_text() == "Save"
+    assert episode.success, episode.render()

tests/test_verify.py

@@ -0,0 +1,99 @@
+"""Success must be evidence, not a claim."""
+
+import pytest
+
+from tiny_computer_agent import (
+    Action, AgentLoop, DryRunExecutor, Element, MockBrain, Observation, Policy,
+)
+
+
+class ScriptedTextScreen:
+    """Screen whose *text* changes, so a verifier has something to see."""
+
+    def __init__(self, texts):
+        self.texts = list(texts)
+        self.captures = 0
+
+    def capture(self) -> Observation:
+        text = self.texts[min(self.captures, len(self.texts) - 1)]
+        self.captures += 1
+        return Observation(index=self.captures - 1, elements=(Element("Save"),), text=text)
+
+
+WAITS = [{"kind": "wait", "seconds": 0.1}] * 4
+# A script longer than the budget, so the budget - not a scripted "done" - ends the run.
+ENDLESS_WAITS = WAITS * 3
+
+
+def run_with_budget(screen, brain, verify=None, budget=3):
+    return AgentLoop(screen=screen, brain=brain, policy=Policy(step_budget=budget),
+                     verify=verify).run("save")
+
+
+def test_without_a_verifier_a_successful_action_can_look_like_a_failure():
+    """The bug this feature exists for: the click worked, the episode still failed."""
+    changing = ScriptedTextScreen(["one", "two", "three", "four"])
+    episode = run_with_budget(changing, MockBrain(ENDLESS_WAITS))
+    assert episode.success is False
+    assert "budget" in episode.stop_reason
+
+
+def test_a_verifier_turns_evidence_into_success():
+    screen = ScriptedTextScreen(["Unsaved changes", "before", "Saved"])
+    episode = AgentLoop(
+        screen=screen, brain=MockBrain(WAITS), verify=lambda obs: "saved" in obs.text.lower()
+    ).run("save the document")
+
+    assert episode.success is True
+    assert episode.stop_reason == "verified after 2 action(s)"
+    assert len(episode.steps) == 2
+
+
+def test_a_verifier_that_never_matches_leaves_it_a_failure():
+    episode = AgentLoop(
+        screen=ScriptedTextScreen(["nothing to see"]), brain=MockBrain(WAITS),
+        verify=lambda obs: "saved" in obs.text.lower(),
+    ).run("save")
+    assert episode.success is False
+
+
+def test_a_broken_verifier_cannot_fake_a_success():
+    def explode(observation):
+        raise RuntimeError("verifier bug")
+
+    episode = run_with_budget(
+        ScriptedTextScreen(["one", "two", "three", "four"]), MockBrain(ENDLESS_WAITS), verify=explode
+    )
+    assert episode.success is False
+    assert "budget" in episode.stop_reason
+
+
+def test_a_model_claiming_done_still_wins_when_there_is_no_verifier():
+    """Unchanged behaviour: with no verifier the brain's "done" is all we have."""
+    episode = AgentLoop(
+        screen=ScriptedTextScreen(["x"]),
+        brain=MockBrain([{"kind": "click", "label": "Save"}, {"kind": "done", "reason": "saved it"}]),
+    ).run("save")
+    assert episode.success is True and "saved it" in episode.stop_reason
+
+
+def test_verify_does_not_disturb_dry_run():
+    executor = DryRunExecutor()
+    AgentLoop(
+        screen=ScriptedTextScreen(["a", "b", "Saved"]), brain=MockBrain(WAITS), executor=executor,
+        verify=lambda obs: "saved" in obs.text.lower(),
+    ).run("save")
+    assert all(a.kind == "wait" for a in executor.performed)
+
+
+def test_cli_verify_text_flag(capsys):
+    from tiny_computer_agent import __main__ as cli
+
+    # the demo screen's text says "Untitled document - unsaved changes"
+    assert cli.main(["--script", '[{"kind": "wait", "seconds": 0.1}]',
+                     "--verify-text", "Unsaved", "--max-steps", "2"]) == 0
+    out = capsys.readouterr().out
+    assert "verified after" in out
+
+    # without it, the same run exhausts its budget and fails
+    assert cli.main(["--script", '[{"kind": "wait", "seconds": 0.1}]', "--max-steps", "1"]) == 1