File size: 3,131 Bytes
627c6cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | """The grid fallback: an action space that needs nothing, and its guard rails."""
import pytest
from PIL import Image
from tiny_computer_agent import Action, Element, Observation, Policy, PolicyViolation
from tiny_computer_agent.perception import (
AutoDetector, GridDetector, OcrDetector, detector_available, make_detector,
)
CELL = Element("cell 12", kind="grid", x=600, y=0, width=60, height=57)
class FixedOcr:
name = "fixed"
def __init__(self, elements=()):
self.elements = tuple(elements)
def detect(self, image):
return self.elements
def test_grid_detector_needs_nothing_at_all():
elements = GridDetector(rows=2, cols=2).detect(Image.new("RGB", (100, 80)))
assert [e.label for e in elements] == ["cell 1", "cell 2", "cell 3", "cell 4"]
assert all(e.kind == "grid" for e in elements)
assert elements[0].center == (25, 20)
assert elements[3].center == (75, 60)
def test_grid_detector_rejects_a_useless_shape():
with pytest.raises(ValueError):
GridDetector(rows=0)
def test_grid_detector_is_always_available_and_default_off():
assert detector_available("grid")[0] is True
assert isinstance(make_detector("grid"), GridDetector)
def test_auto_uses_ocr_when_it_finds_text():
detector = AutoDetector(ocr=FixedOcr((Element("Save"),)), grid=GridDetector(rows=1, cols=1))
assert [e.label for e in detector.detect(Image.new("RGB", (10, 10)))] == ["Save"]
def test_auto_falls_back_to_the_grid_when_ocr_finds_nothing():
detector = AutoDetector(ocr=FixedOcr(()), grid=GridDetector(rows=1, cols=2))
assert [e.label for e in detector.detect(Image.new("RGB", (10, 10)))] == ["cell 1", "cell 2"]
def test_a_grid_click_is_fine_while_planning():
Policy(dry_run=True).check(Action("click", label="cell 12"), Observation(elements=(CELL,)))
def test_a_grid_click_is_refused_with_real_input():
"""A blind click on a real desktop must be a deliberate choice, not a default."""
with pytest.raises(PolicyViolation, match="blind click"):
Policy(dry_run=False).check(Action("click", label="cell 12"), Observation(elements=(CELL,)))
def test_a_grid_click_is_refused_even_when_the_index_is_stale():
with pytest.raises(PolicyViolation, match="blind click"):
Policy(dry_run=False).check(Action("click", label="cell 47"), Observation())
def test_a_grid_click_is_allowed_when_asked_for():
Policy(dry_run=False, allow_grid_clicks=True).check(
Action("click", label="cell 12"), Observation(elements=(CELL,))
)
def test_an_ordinary_click_is_unaffected_by_the_grid_guard():
Policy(dry_run=False).check(
Action("click", label="Save"),
Observation(elements=(Element("Save", x=1, y=2, width=3, height=4),)),
)
@pytest.mark.skipif(not detector_available("ocr")[0], reason="no OCR engine installed")
def test_real_ocr_beats_the_grid_on_a_real_screenshot():
from tiny_computer_agent.demo import fake_desktop
elements = AutoDetector().detect(fake_desktop())
assert elements and any(e.kind != "grid" for e in elements)
assert isinstance(OcrDetector(), OcrDetector)
|