| """The grid fallback: an action space that needs nothing, and its guard rails.""" |
|
|
| import pytest |
| from PIL import Image |
|
|
| from tiny_computer_agent import Action, Element, Observation, Policy, PolicyViolation |
| from tiny_computer_agent.perception import ( |
| AutoDetector, GridDetector, OcrDetector, detector_available, make_detector, |
| ) |
|
|
| CELL = Element("cell 12", kind="grid", x=600, y=0, width=60, height=57) |
|
|
|
|
| class FixedOcr: |
| name = "fixed" |
|
|
| def __init__(self, elements=()): |
| self.elements = tuple(elements) |
|
|
| def detect(self, image): |
| return self.elements |
|
|
|
|
| def test_grid_detector_needs_nothing_at_all(): |
| elements = GridDetector(rows=2, cols=2).detect(Image.new("RGB", (100, 80))) |
| assert [e.label for e in elements] == ["cell 1", "cell 2", "cell 3", "cell 4"] |
| assert all(e.kind == "grid" for e in elements) |
| assert elements[0].center == (25, 20) |
| assert elements[3].center == (75, 60) |
|
|
|
|
| def test_grid_detector_rejects_a_useless_shape(): |
| with pytest.raises(ValueError): |
| GridDetector(rows=0) |
|
|
|
|
| def test_grid_detector_is_always_available_and_default_off(): |
| assert detector_available("grid")[0] is True |
| assert isinstance(make_detector("grid"), GridDetector) |
|
|
|
|
| def test_auto_uses_ocr_when_it_finds_text(): |
| detector = AutoDetector(ocr=FixedOcr((Element("Save"),)), grid=GridDetector(rows=1, cols=1)) |
| assert [e.label for e in detector.detect(Image.new("RGB", (10, 10)))] == ["Save"] |
|
|
|
|
| def test_auto_falls_back_to_the_grid_when_ocr_finds_nothing(): |
| detector = AutoDetector(ocr=FixedOcr(()), grid=GridDetector(rows=1, cols=2)) |
| assert [e.label for e in detector.detect(Image.new("RGB", (10, 10)))] == ["cell 1", "cell 2"] |
|
|
|
|
| def test_a_grid_click_is_fine_while_planning(): |
| Policy(dry_run=True).check(Action("click", label="cell 12"), Observation(elements=(CELL,))) |
|
|
|
|
| def test_a_grid_click_is_refused_with_real_input(): |
| """A blind click on a real desktop must be a deliberate choice, not a default.""" |
| with pytest.raises(PolicyViolation, match="blind click"): |
| Policy(dry_run=False).check(Action("click", label="cell 12"), Observation(elements=(CELL,))) |
|
|
|
|
| def test_a_grid_click_is_refused_even_when_the_index_is_stale(): |
| with pytest.raises(PolicyViolation, match="blind click"): |
| Policy(dry_run=False).check(Action("click", label="cell 47"), Observation()) |
|
|
|
|
| def test_a_grid_click_is_allowed_when_asked_for(): |
| Policy(dry_run=False, allow_grid_clicks=True).check( |
| Action("click", label="cell 12"), Observation(elements=(CELL,)) |
| ) |
|
|
|
|
| def test_an_ordinary_click_is_unaffected_by_the_grid_guard(): |
| Policy(dry_run=False).check( |
| Action("click", label="Save"), |
| Observation(elements=(Element("Save", x=1, y=2, width=3, height=4),)), |
| ) |
|
|
|
|
| @pytest.mark.skipif(not detector_available("ocr")[0], reason="no OCR engine installed") |
| def test_real_ocr_beats_the_grid_on_a_real_screenshot(): |
| from tiny_computer_agent.demo import fake_desktop |
|
|
| elements = AutoDetector().detect(fake_desktop()) |
| assert elements and any(e.kind != "grid" for e in elements) |
| assert isinstance(OcrDetector(), OcrDetector) |
|
|
