import pytest from tiny_computer_agent import Element, FakeScreen, Observation from tiny_computer_agent.demo import fake_desktop, write_fake_desktop from tiny_computer_agent.observe import ImageScreen from tiny_computer_agent.perception import ( DetectorUnavailable, IndexedScreen, OcrDetector, detector_available, make_detector, ) BOX = [[10, 10], [80, 10], [80, 30], [10, 30]] class FakeDetector: name = "fake" def __init__(self, elements=(), error=None): self.elements = tuple(elements) self.error = error def detect(self, image): if self.error is not None: raise self.error return self.elements def test_low_confidence_boxes_are_dropped(): assert OcrDetector(min_score=0.9)._to_element(BOX, "Save", 0.5) is None def test_prose_fragments_and_punctuation_are_not_widgets(): detector = OcrDetector(max_label_chars=20, min_label_chars=2) assert detector._to_element(BOX, "a long sentence that is not a widget", 1.0) is None assert detector._to_element(BOX, "S", 1.0) is None assert detector._to_element(BOX, " ... ", 1.0) is None assert detector._to_element(BOX, " first ", 1.0).label == "first" def test_a_box_becomes_a_clickable_element(): element = OcrDetector()._to_element(BOX, "Save", 0.99) assert (element.x, element.y, element.width, element.height) == (10, 10, 70, 20) assert element.center == (45, 20) def test_indexed_screen_replaces_elements_with_detected_ones(): screen = IndexedScreen( FakeScreen([(Element("stale"),)]), FakeDetector((Element("Save"), Element("Cancel"))) ) observation = screen.capture() assert [e.label for e in observation.elements] == ["Save", "Cancel"] assert observation.text == "Save Cancel" # text filled from the index def test_indexed_screen_keeps_text_from_the_source(): screen = IndexedScreen(FakeScreen([(Element("x"),)], text="hello"), FakeDetector((Element("Save"),))) assert screen.capture().text == "hello" def test_a_perception_error_does_not_kill_the_run(): screen = IndexedScreen(FakeScreen([(Element("x"),)]), FakeDetector(error=ValueError("boom"))) observation = screen.capture() assert observation.elements == () assert "perception error" in observation.text def test_a_missing_engine_is_reported_loudly(): screen = IndexedScreen(FakeScreen([(Element("x"),)]), FakeDetector(error=DetectorUnavailable("no engine"))) with pytest.raises(DetectorUnavailable): screen.capture() def test_unknown_detector_names_are_reported(): available, detail = detector_available("nope") assert available is False and "unknown detector" in detail with pytest.raises(DetectorUnavailable): make_detector("nope") def test_image_screen_replays_a_still_and_holds_it(tmp_path): path = write_fake_desktop(tmp_path / "screen.png") screen = ImageScreen(str(path)) assert screen.capture().frame.size == screen.capture().frame.size assert isinstance(screen.capture().text, str) def test_image_screen_rejects_something_empty(): with pytest.raises(ValueError): ImageScreen([]) def test_the_synthetic_desktop_looks_like_the_intended_screen(): image = fake_desktop() assert image.size == (720, 460) @pytest.mark.skipif(not detector_available("ocr")[0], reason="no OCR engine installed") # conftest.require_ocr is the in-test form def test_real_ocr_reads_a_real_screenshot(): """The whole point: pixels in, labelled elements with usable coordinates out.""" elements = OcrDetector().detect(fake_desktop()) labels = [e.label for e in elements] assert len(elements) >= 3, labels assert any(e.matches("Save") for e in elements), labels save = next(e for e in elements if e.matches("Save")) # the Save button is drawn at (200..380, 265..300) assert 200 <= save.center[0] <= 380 assert 260 <= save.center[1] <= 310 @pytest.mark.skipif(not detector_available("ocr")[0], reason="no OCR engine installed") # conftest.require_ocr is the in-test form def test_ocr_index_plugs_straight_into_a_click(): """Label -> pixel resolution on coordinates the detector actually produced.""" from tiny_computer_agent import Action, resolve_point observation = Observation(elements=OcrDetector().detect(fake_desktop())) point = resolve_point(Action("click", label="Save"), observation) assert point is not None assert 200 <= point[0] <= 380