Blame for tests/test_accessibility.py

"""The accessibility-tree detector: exact elements, and honest failures.

Nothing here needs a desktop. The tree is injected through `TreeReader`, so the
filtering, the geometry and the coordinate contract are all tested for real; the
one test that drives actual AT-SPI is opt-in, like the live-desktop tests.
"""

import os

import pytest

from conftest import require_atspi
from PIL import Image

import platform

from tiny_computer_agent import (
    INTERACTIVE_ROLES, TEXT_ROLES, AccessibleNode, AtspiDetector, AtspiReader,
    AutoDetector, Element, GridDetector, IndexedScreen, Observation, UiaDetector,
    UiaReader, detector_available, make_detector,
)
from tiny_computer_agent.accessibility import accessibility_available
from tiny_computer_agent.perception import DetectorUnavailable


class FakeReader:
    """A tree, exactly as AT-SPI would report it."""

    name = "fake"

    def __init__(self, nodes):
        self.nodes = tuple(nodes)
        self.reads = 0

    def read(self):
        self.reads += 1
        return self.nodes


FRAME = Image.new("RGB", (1000, 700), (20, 22, 28))     # a frame, for grid fallback


def node(role, name, x=100, y=100, width=100, height=36, depth=2):
    return AccessibleNode(role, name, x, y, width, height, depth)


TREE = [
    node("application", "gtk_dialog.py", 0, 0, 0, 0, 0),
    node("frame", "Accessible desktop", 160, 160, 460, 220, 1),
    node("panel", "", 174, 174, 432, 192, 2),
    node("button", "Files", 174, 201, 100, 36),
    node("button", "Terminal", 282, 201, 100, 36),
    node("button", "Save", 390, 201, 100, 36),
    node("button", "Cancel", 498, 201, 100, 36),
    node("label", "Choose an action", 174, 245, 424, 19),
    node("label", "last action: (none)", 174, 174, 424, 19),
]


# -- what the tree gives us --------------------------------------------------

def test_the_tree_yields_labelled_widgets_with_real_coordinates():
    elements = AtspiDetector(reader=FakeReader(TREE)).detect()
    assert [(e.label, e.kind) for e in elements] == [
        ("Files", "button"), ("Terminal", "button"), ("Save", "button"), ("Cancel", "button"),
    ]
    save = next(e for e in elements if e.label == "Save")
    assert (save.x, save.y, save.width, save.height) == (390, 201, 100, 36)
    assert save.center == (440, 219)


def test_window_furniture_is_not_offered_as_a_choice():
    """A frame and a panel have names too; giving them to a model is a trap."""
    roles = {e.kind for e in AtspiDetector(reader=FakeReader(TREE)).detect()}
    assert "frame" not in roles and "panel" not in roles and "application" not in roles
    assert roles <= INTERACTIVE_ROLES


def test_labels_can_be_read_without_being_clickable():
    plain = AtspiDetector(reader=FakeReader(TREE))
    with_text = AtspiDetector(reader=FakeReader(TREE), include_text=True)
    assert not any(e.kind == "label" for e in plain.detect())
    assert [e.label for e in with_text.detect() if e.kind == "label"] == [
        "Choose an action", "last action: (none)"]
    assert "Choose an action" in with_text.text()


def test_labels_are_filtered_by_length_and_size():
    tree = [node("button", "x"), node("button", "a" * 80), node("button", "Sneaky", 5, 5, 2, 2)]

    strict = AtspiDetector(reader=FakeReader(tree), min_label_chars=2)
    assert [e.label for e in strict.detect()] == []      # too short, too long, too small

    loose = AtspiDetector(reader=FakeReader(tree), min_label_chars=1,
                          max_label_chars=6, min_size=1)
    assert [e.label for e in loose.detect()] == ["x", "Sneaky"]   # 80 chars still too long


def test_off_screen_and_duplicate_nodes_are_dropped():
    tree = [node("button", "Save"), node("button", "Save"),
            node("button", "Ghost", -400, -400)]
    assert [e.label for e in AtspiDetector(reader=FakeReader(tree)).detect()] == ["Save"]


def test_the_detector_ignores_the_image_it_is_handed():
    reader = FakeReader(TREE)
    detector = AtspiDetector(reader=reader)
    detector.detect(None)
    detector.detect(object())                 # any object at all
    assert reader.reads == 2


# -- the coordinate contract -------------------------------------------------

class ScaledSource:
    """A source whose frames are downscaled: every element needs re-mapping."""

    def __init__(self):
        self.origin = (0, 0)
        self.frame_scale = 2.0

    def capture(self):
        return Observation(index=0, frame=FRAME)

    def map_to_screen(self, element):
        return Element(label=element.label, kind=element.kind, x=element.x * 2,
                       y=element.y * 2, width=element.width * 2, height=element.height * 2)


def test_screen_coordinates_are_not_double_mapped_by_a_downscaled_capture():
    """AT-SPI already speaks screen pixels; a downscale must not scale them again."""
    detector = AtspiDetector(reader=FakeReader([node("button", "Save", 390, 201, 100, 36)]))
    observation = IndexedScreen(ScaledSource(), detector).capture()
    save = observation.elements[0]
    assert (save.x, save.y) == (390, 201), "the accessibility tree was scaled twice"


def test_the_chain_reports_the_space_of_the_branch_it_used():
    """The bug this pins down: `best` wraps the tree detector, so a *static* flag
    said "frame" and every accessibility click was re-mapped through the downscale
    - shifting it by the capture's scale factor, silently."""
    tree = AtspiDetector(reader=FakeReader([node("button", "Save", 390, 201, 100, 36)]))

    class Empty:
        name = "empty"

        def detect(self, image=None):
            return ()

    chained = AutoDetector(accessibility=tree, grid=GridDetector(rows=1, cols=2))
    observation = IndexedScreen(ScaledSource(), chained).capture()
    assert observation.elements[0].x == 390, "the chained tree index was double-mapped"

    chained.accessibility = Empty()                      # now the grid answers
    observation = IndexedScreen(ScaledSource(), chained).capture()
    assert observation.elements[0].x == 0                # frame space: mapped
    assert chained.coordinates_of_last_detect() == "frame"


def test_a_frame_relative_detector_is_still_mapped():
    """The opt-out is per-detector: a grid in a downscaled frame must be mapped."""
    from tiny_computer_agent import GridDetector
    observation = IndexedScreen(ScaledSource(), GridDetector(rows=1, cols=2)).capture()
    assert [e.x for e in observation.elements] == [0, 1000]   # 500 * frame_scale of 2


# -- honest failures --------------------------------------------------------

def test_a_reader_that_cannot_reach_the_bus_raises_rather_than_lying():
    class Broken:
        name = "broken"

        def read(self):
            raise DetectorUnavailable("no a11y bus")

    with pytest.raises(DetectorUnavailable):
        AtspiDetector(reader=Broken()).detect()


def test_an_unknown_accessibility_backend_is_refused():
    assert accessibility_available("telepathy")[0] is False


def test_the_reader_itself_works_without_a_desktop_at_all():
    """AT-SPI is present here, so reading an empty desktop returns no nodes."""
    available, detail = accessibility_available("atspi")
    if not available:
        pytest.skip(f"no AT-SPI here: {detail}")
    assert isinstance(AtspiReader().read(), tuple)


# -- the fallback chain ------------------------------------------------------

def test_best_uses_the_tree_when_it_has_content():
    chains = []

    class RecordingAccessibility:
        name = "a11y"

        def detect(self, image=None):
            chains.append("accessibility")
            return (Element("Save", kind="button", x=390, y=201, width=100, height=36),)

    class RecordingOcr:
        name = "ocr"

        def detect(self, image):
            chains.append("ocr")
            return (Element("Guessed"),)

    detector = AutoDetector(ocr=RecordingOcr(), accessibility=RecordingAccessibility())
    assert [e.label for e in detector.detect(None)] == ["Save"]
    assert chains == ["accessibility"]


def test_best_falls_through_to_ocr_then_the_grid():
    from tiny_computer_agent import GridDetector

    class EmptyTree:
        name = "a11y"

        def detect(self, image=None):
            return ()

    class BrokenOcr:
        name = "ocr"

        def detect(self, image):
            raise DetectorUnavailable("no engine")

    with_ocr = AutoDetector(ocr=type("Ocr", (), {
        "detect": lambda self, image: (Element("From OCR"),)})(), accessibility=EmptyTree())
    assert [e.label for e in with_ocr.detect(FRAME)] == ["From OCR"]

    without_ocr = AutoDetector(ocr=BrokenOcr(), grid=GridDetector(rows=1, cols=2),
                               accessibility=EmptyTree())
    assert [e.label for e in without_ocr.detect(FRAME)] == ["cell 1", "cell 2"]


def test_the_auto_detector_still_defaults_to_ocr_then_grid():
    """The new chain is opt-in: `auto` behaviour must not change."""
    assert AutoDetector().accessibility is None


def test_the_new_detector_names_are_reachable():
    assert type(make_detector("best")).__name__ == "AutoDetector"
    assert TEXT_ROLES and "label" in TEXT_ROLES
    require_atspi()                     # the atspi name needs the typelib to build
    assert type(make_detector("atspi")).__name__ == "AtspiDetector"
    ok, detail = detector_available("atspi")
    assert isinstance(ok, bool) and detail


# -- the Windows backend: its mapping, exercised without Windows ---------------

class FakeBox:
    def __init__(self, left, top, right, bottom):
        self.left, self.top, self.right, self.bottom = left, top, right, bottom


class FakeControl:
    """The slice of a UI Automation control element this reader actually uses."""

    def __init__(self, name, control_type, box=None, children=()):
        self.Name = name
        self.ControlTypeName = control_type
        self.BoundingRectangle = box
        self._children = list(children)

    def GetChildren(self):
        return self._children


def uia_tree():
    """The same window the GTK app opens, as UI Automation would report it."""
    return FakeControl("Accessible desktop", "WindowControl", FakeBox(160, 160, 620, 380), [
        FakeControl("", "PaneControl", FakeBox(174, 174, 606, 366), [
            FakeControl("Files", "ButtonControl", FakeBox(174, 201, 274, 237)),
            FakeControl("Terminal", "ButtonControl", FakeBox(282, 201, 382, 237)),
            FakeControl("Save", "ButtonControl", FakeBox(390, 201, 490, 237)),
            FakeControl("Cancel", "ButtonControl", FakeBox(498, 201, 598, 237)),
            FakeControl("Choose an action", "TextControl", FakeBox(174, 245, 598, 264)),
        ]),
    ])


def uia_elements():
    return UiaDetector(reader=UiaReader(provider=uia_tree)).detect()


def test_uia_maps_control_types_to_the_roles_the_harness_speaks():
    assert [(e.label, e.kind) for e in uia_elements()] == [
        ("Files", "button"), ("Terminal", "button"), ("Save", "button"), ("Cancel", "button")]
    save = next(e for e in uia_elements() if e.label == "Save")
    assert (save.x, save.y, save.width, save.height) == (390, 201, 100, 36)
    assert save.center == (440, 219)


def test_uia_window_furniture_is_not_offered_as_a_choice():
    """A window and a pane have names too, and neither is a button."""
    roles = {e.kind for e in uia_elements()}
    assert roles == {"button"}


def test_uia_keeps_an_unnamed_container_but_walks_into_it():
    """The pane has no name, so it is not an element - its children still are."""
    assert all(e.label for e in uia_elements())
    assert len(uia_elements()) == 4


def test_uia_respects_the_depth_and_node_caps():
    deep = UiaReader(provider=uia_tree, max_depth=1)
    assert [e.label for e in UiaDetector(reader=deep).detect()] == []
    # the window itself is the first node, so the cap of two leaves one button
    capped = UiaReader(provider=uia_tree, max_nodes=2)
    assert [e.label for e in UiaDetector(reader=capped).detect()] == ["Files"]


def test_uia_detector_text_reads_the_tree_too():
    assert "Choose an action" in UiaDetector(reader=UiaReader(provider=uia_tree)).text()


def test_the_windows_backend_is_reported_honestly_on_this_machine():
    ok, detail = accessibility_available("uia")
    if platform.system() == "Windows":
        return                      # on Windows this is a real, working backend
    assert ok is False and "Windows-only" in detail


def test_the_uia_reader_refuses_rather_than_returning_an_empty_index():
    if platform.system() == "Windows":
        pytest.skip("this machine really is Windows")
    with pytest.raises(DetectorUnavailable, match="Windows-only"):
        UiaReader().read()


def test_the_windows_detector_name_is_reachable_without_windows():
    """The Windows backend's *mapping* is exercised anywhere; its tree is not."""
    ok, detail = detector_available("uia")
    assert isinstance(ok, bool) and detail


def test_the_detector_names_cover_both_backends():
    require_atspi()
    assert type(make_detector("atspi")).__name__ == "AtspiDetector"


def test_a_provider_that_throws_is_reported_as_unavailable():
    def broken():
        raise RuntimeError("COM is not initialised")

    with pytest.raises(DetectorUnavailable, match="could not reach UI Automation"):
        UiaReader(provider=broken).read()


# -- the one test that drives real AT-SPI ------------------------------------

OPT_IN = os.environ.get("TINY_COMPUTER_AGENT_LIVE_TEST") == "1"


@pytest.mark.skipif(not OPT_IN, reason="starts a real GTK window under a session bus")
def test_atspi_reads_a_real_gtk_button_at_the_right_pixels(tmp_path):
    import pathlib
    import subprocess
    import sys
    import time

    if not accessibility_available("atspi")[0]:
        pytest.skip("no AT-SPI on this machine")
    if not shutil_which("dbus-daemon"):
        pytest.skip("no dbus-daemon")

    address = subprocess.run(["dbus-daemon", "--session", "--fork", "--print-address=1"],
                             capture_output=True, text=True).stdout.strip()
    os.environ["DBUS_SESSION_BUS_ADDRESS"] = address
    os.environ.setdefault("DISPLAY", ":99")
    os.environ["NO_AT_BRIDGE"] = "0"

    ready = pathlib.Path(tmp_path) / "gtk.ready"
    app = subprocess.Popen(
        [sys.executable, "showcase_scenes/gtk_dialog.py", str(tmp_path / "marker"), str(ready)])
    try:
        deadline = time.time() + 25
        while time.time() < deadline and not ready.exists():
            if app.poll() is not None:
                pytest.fail("the GTK window exited early")
            time.sleep(0.25)
        assert ready.exists(), "the GTK window never mapped"
        time.sleep(3)

        elements = AtspiDetector(reader=AtspiReader()).detect()
        labels = [e.label for e in elements]
        assert "Save" in labels, f"the tree had no Save button: {labels}"
        save = next(e for e in elements if e.label == "Save")
        # the window is at +160+160 and the buttons are 100x36, laid out left to right
        assert 160 < save.x < 160 + 460 and 160 < save.y < 160 + 220, save
        assert (save.width, save.height) == (100, 36)
    finally:
        app.terminate()
        app.wait(timeout=10)


def shutil_which(name):
    import shutil
    return shutil.which(name)