| import pytest |
|
|
| from tiny_computer_agent import Element, FakeScreen, Observation |
| from tiny_computer_agent.demo import fake_desktop, write_fake_desktop |
| from tiny_computer_agent.observe import ImageScreen |
| from tiny_computer_agent.perception import ( |
| DetectorUnavailable, IndexedScreen, OcrDetector, detector_available, make_detector, |
| ) |
|
|
| BOX = [[10, 10], [80, 10], [80, 30], [10, 30]] |
|
|
|
|
| class FakeDetector: |
| name = "fake" |
|
|
| def __init__(self, elements=(), error=None): |
| self.elements = tuple(elements) |
| self.error = error |
|
|
| def detect(self, image): |
| if self.error is not None: |
| raise self.error |
| return self.elements |
|
|
|
|
| def test_low_confidence_boxes_are_dropped(): |
| assert OcrDetector(min_score=0.9)._to_element(BOX, "Save", 0.5) is None |
|
|
|
|
| def test_prose_fragments_and_punctuation_are_not_widgets(): |
| detector = OcrDetector(max_label_chars=20, min_label_chars=2) |
| assert detector._to_element(BOX, "a long sentence that is not a widget", 1.0) is None |
| assert detector._to_element(BOX, "S", 1.0) is None |
| assert detector._to_element(BOX, " ... ", 1.0) is None |
| assert detector._to_element(BOX, " first ", 1.0).label == "first" |
|
|
|
|
| def test_a_box_becomes_a_clickable_element(): |
| element = OcrDetector()._to_element(BOX, "Save", 0.99) |
| assert (element.x, element.y, element.width, element.height) == (10, 10, 70, 20) |
| assert element.center == (45, 20) |
|
|
|
|
| def test_indexed_screen_replaces_elements_with_detected_ones(): |
| screen = IndexedScreen( |
| FakeScreen([(Element("stale"),)]), FakeDetector((Element("Save"), Element("Cancel"))) |
| ) |
| observation = screen.capture() |
| assert [e.label for e in observation.elements] == ["Save", "Cancel"] |
| assert observation.text == "Save Cancel" |
|
|
|
|
| def test_indexed_screen_keeps_text_from_the_source(): |
| screen = IndexedScreen(FakeScreen([(Element("x"),)], text="hello"), |
| FakeDetector((Element("Save"),))) |
| assert screen.capture().text == "hello" |
|
|
|
|
| def test_a_perception_error_does_not_kill_the_run(): |
| screen = IndexedScreen(FakeScreen([(Element("x"),)]), FakeDetector(error=ValueError("boom"))) |
| observation = screen.capture() |
| assert observation.elements == () |
| assert "perception error" in observation.text |
|
|
|
|
| def test_a_missing_engine_is_reported_loudly(): |
| screen = IndexedScreen(FakeScreen([(Element("x"),)]), |
| FakeDetector(error=DetectorUnavailable("no engine"))) |
| with pytest.raises(DetectorUnavailable): |
| screen.capture() |
|
|
|
|
| def test_unknown_detector_names_are_reported(): |
| available, detail = detector_available("nope") |
| assert available is False and "unknown detector" in detail |
| with pytest.raises(DetectorUnavailable): |
| make_detector("nope") |
|
|
|
|
| def test_image_screen_replays_a_still_and_holds_it(tmp_path): |
| path = write_fake_desktop(tmp_path / "screen.png") |
| screen = ImageScreen(str(path)) |
| assert screen.capture().frame.size == screen.capture().frame.size |
| assert isinstance(screen.capture().text, str) |
|
|
|
|
| def test_image_screen_rejects_something_empty(): |
| with pytest.raises(ValueError): |
| ImageScreen([]) |
|
|
|
|
| def test_the_synthetic_desktop_looks_like_the_intended_screen(): |
| image = fake_desktop() |
| assert image.size == (720, 460) |
|
|
|
|
| @pytest.mark.skipif(not detector_available("ocr")[0], reason="no OCR engine installed") |
| def test_real_ocr_reads_a_real_screenshot(): |
| """The whole point: pixels in, labelled elements with usable coordinates out.""" |
| elements = OcrDetector().detect(fake_desktop()) |
| labels = [e.label for e in elements] |
| assert len(elements) >= 3, labels |
| assert any(e.matches("Save") for e in elements), labels |
|
|
| save = next(e for e in elements if e.matches("Save")) |
| |
| assert 200 <= save.center[0] <= 380 |
| assert 260 <= save.center[1] <= 310 |
|
|
|
|
| @pytest.mark.skipif(not detector_available("ocr")[0], reason="no OCR engine installed") |
| def test_ocr_index_plugs_straight_into_a_click(): |
| """Label -> pixel resolution on coordinates the detector actually produced.""" |
| from tiny_computer_agent import Action, resolve_point |
|
|
| observation = Observation(elements=OcrDetector().detect(fake_desktop())) |
| point = resolve_point(Action("click", label="Save"), observation) |
| assert point is not None |
| assert 200 <= point[0] <= 380 |
|
|
