File size: 4,513 Bytes
627c6cf 16e3e7d 627c6cf 16e3e7d 627c6cf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | import pytest
from tiny_computer_agent import Element, FakeScreen, Observation
from tiny_computer_agent.demo import fake_desktop, write_fake_desktop
from tiny_computer_agent.observe import ImageScreen
from tiny_computer_agent.perception import (
DetectorUnavailable, IndexedScreen, OcrDetector, detector_available, make_detector,
)
BOX = [[10, 10], [80, 10], [80, 30], [10, 30]]
class FakeDetector:
name = "fake"
def __init__(self, elements=(), error=None):
self.elements = tuple(elements)
self.error = error
def detect(self, image):
if self.error is not None:
raise self.error
return self.elements
def test_low_confidence_boxes_are_dropped():
assert OcrDetector(min_score=0.9)._to_element(BOX, "Save", 0.5) is None
def test_prose_fragments_and_punctuation_are_not_widgets():
detector = OcrDetector(max_label_chars=20, min_label_chars=2)
assert detector._to_element(BOX, "a long sentence that is not a widget", 1.0) is None
assert detector._to_element(BOX, "S", 1.0) is None
assert detector._to_element(BOX, " ... ", 1.0) is None
assert detector._to_element(BOX, " first ", 1.0).label == "first"
def test_a_box_becomes_a_clickable_element():
element = OcrDetector()._to_element(BOX, "Save", 0.99)
assert (element.x, element.y, element.width, element.height) == (10, 10, 70, 20)
assert element.center == (45, 20)
def test_indexed_screen_replaces_elements_with_detected_ones():
screen = IndexedScreen(
FakeScreen([(Element("stale"),)]), FakeDetector((Element("Save"), Element("Cancel")))
)
observation = screen.capture()
assert [e.label for e in observation.elements] == ["Save", "Cancel"]
assert observation.text == "Save Cancel" # text filled from the index
def test_indexed_screen_keeps_text_from_the_source():
screen = IndexedScreen(FakeScreen([(Element("x"),)], text="hello"),
FakeDetector((Element("Save"),)))
assert screen.capture().text == "hello"
def test_a_perception_error_does_not_kill_the_run():
screen = IndexedScreen(FakeScreen([(Element("x"),)]), FakeDetector(error=ValueError("boom")))
observation = screen.capture()
assert observation.elements == ()
assert "perception error" in observation.text
def test_a_missing_engine_is_reported_loudly():
screen = IndexedScreen(FakeScreen([(Element("x"),)]),
FakeDetector(error=DetectorUnavailable("no engine")))
with pytest.raises(DetectorUnavailable):
screen.capture()
def test_unknown_detector_names_are_reported():
available, detail = detector_available("nope")
assert available is False and "unknown detector" in detail
with pytest.raises(DetectorUnavailable):
make_detector("nope")
def test_image_screen_replays_a_still_and_holds_it(tmp_path):
path = write_fake_desktop(tmp_path / "screen.png")
screen = ImageScreen(str(path))
assert screen.capture().frame.size == screen.capture().frame.size
assert isinstance(screen.capture().text, str)
def test_image_screen_rejects_something_empty():
with pytest.raises(ValueError):
ImageScreen([])
def test_the_synthetic_desktop_looks_like_the_intended_screen():
image = fake_desktop()
assert image.size == (720, 460)
@pytest.mark.skipif(not detector_available("ocr")[0], reason="no OCR engine installed") # conftest.require_ocr is the in-test form
def test_real_ocr_reads_a_real_screenshot():
"""The whole point: pixels in, labelled elements with usable coordinates out."""
elements = OcrDetector().detect(fake_desktop())
labels = [e.label for e in elements]
assert len(elements) >= 3, labels
assert any(e.matches("Save") for e in elements), labels
save = next(e for e in elements if e.matches("Save"))
# the Save button is drawn at (200..380, 265..300)
assert 200 <= save.center[0] <= 380
assert 260 <= save.center[1] <= 310
@pytest.mark.skipif(not detector_available("ocr")[0], reason="no OCR engine installed") # conftest.require_ocr is the in-test form
def test_ocr_index_plugs_straight_into_a_click():
"""Label -> pixel resolution on coordinates the detector actually produced."""
from tiny_computer_agent import Action, resolve_point
observation = Observation(elements=OcrDetector().detect(fake_desktop()))
point = resolve_point(Action("click", label="Save"), observation)
assert point is not None
assert 200 <= point[0] <= 380
|