| import pytest |
|
|
| from tiny_computer_agent import ( |
| Action, BrainError, DecisionMenu, Element, GoalUnreachable, ModelBrain, |
| Observation, extract_json_object, extract_json_objects, parse_decision, |
| ) |
|
|
| FRAME = Observation(elements=( |
| Element("Files", x=10, y=5, width=40, height=20), |
| Element("Save", x=200, y=260, width=60, height=20), |
| Element("Cancel", x=300, y=260, width=60, height=20), |
| ), text="untitled document") |
|
|
|
|
| def test_menu_is_numbered_in_reading_order_and_capped(): |
| observation = Observation(elements=tuple( |
| Element(f"b{i}", x=(i % 2) * 100, y=(i // 2) * 50) for i in range(6) |
| )) |
| menu = DecisionMenu(observation, max_items=3) |
| assert [item.index for item in menu] == [1, 2, 3] |
| assert len(menu) == 3 and bool(menu) |
| assert "1." in menu.render() |
|
|
|
|
| def test_menu_lookup_tolerates_what_models_send(): |
| menu = DecisionMenu(FRAME) |
| assert menu.item(1).label == "Files" |
| assert menu.item("2").label == "Save" |
| assert menu.item(2.0).label == "Save" |
| assert menu.item(0) is None and menu.item(99) is None and menu.item("x") is None |
|
|
|
|
| def test_menu_resolves_labels_through_normalisation(): |
| menu = DecisionMenu(Observation(elements=(Element("Save ."),))) |
| assert menu.find("save").label == "Save ." |
| assert menu.find("nothing").__class__ is type(None) |
|
|
|
|
| def test_json_objects_are_extracted_in_order_and_the_last_wins(): |
| raw = 'First {"kind": "click", "index": 1} then {"kind": "done", "reason": "ok"}' |
| assert [o["kind"] for o in extract_json_objects(raw)] == ["click", "done"] |
| |
| assert extract_json_object(raw)["kind"] == "done" |
| with pytest.raises(BrainError, match="no JSON object"): |
| extract_json_object("no json here") |
|
|
|
|
| def test_parse_decision_json_index(): |
| action = parse_decision('{"kind": "click", "index": 2, "reason": "saves"}', FRAME) |
| assert action.kind == "click" and action.label == "Save" |
|
|
|
|
| def test_parse_decision_echo_of_the_prompt_is_rejected(): |
| echoed = '{"kind": "click", "index": 1} {"kind": "click", "index": 2}' |
| assert parse_decision(echoed, FRAME).label == "Save" |
| with pytest.raises(BrainError, match="echo"): |
| parse_decision(DecisionMenu(FRAME).render(), FRAME) |
|
|
|
|
| def test_parse_decision_bare_number_and_prose_label(): |
| assert parse_decision("2", FRAME).label == "Save" |
| assert parse_decision('Click on "Save" in the dialogue.', FRAME).label == "Save" |
| assert parse_decision("The goal is already done.", FRAME).kind == "done" |
|
|
|
|
| def test_parse_decision_rejects_an_index_outside_the_menu(): |
| with pytest.raises(BrainError, match="not in the"): |
| parse_decision('{"kind": "click", "index": 99}', FRAME) |
|
|
|
|
| def test_parse_decision_rejects_a_label_that_is_not_on_screen(): |
| with pytest.raises(BrainError, match="not on screen"): |
| parse_decision('{"kind": "click", "label": "Format disk"}', FRAME) |
|
|
|
|
| def test_parse_decision_tolerates_a_garbled_verb_when_a_label_is_present(): |
| |
| action = parse_decision('{"kind": "file", "label": "Save"}', FRAME) |
| assert action.kind == "click" and action.label == "Save" |
|
|
|
|
| def test_parse_decision_empty_and_useless_replies(): |
| for raw in ["", " ", "I am not sure what to do, sorry."]: |
| with pytest.raises(BrainError): |
| parse_decision(raw, FRAME) |
|
|
|
|
| class _ScriptedModel(ModelBrain): |
| name = "scripted" |
|
|
| def __init__(self, replies, **kwargs): |
| super().__init__("fake/model", **kwargs) |
| self._replies = list(replies) |
| self.prompts = [] |
|
|
| def _generate(self, prompt, observation): |
| self.prompts.append(prompt) |
| return self._replies.pop(0) |
|
|
|
|
| def test_model_brain_tells_the_model_what_it_already_tried(): |
| brain = _ScriptedModel(['{"kind": "click", "index": 2}', '{"kind": "done", "reason": "ok"}']) |
| first = brain.decide("save", FRAME) |
| assert first.label == "Save" |
| assert "Already attempted" not in brain.prompts[0] |
|
|
| second = brain.decide("save", FRAME) |
| prompt = brain.prompts[1] |
| assert "Already attempted" in prompt |
| assert "screen did NOT change" in prompt |
| assert "Choose a DIFFERENT action" in prompt |
| assert second.kind == "done" |
|
|
|
|
| def test_model_brain_history_is_bounded(): |
| brain = _ScriptedModel(['{"kind": "wait", "seconds": 1}'] * 6) |
| for _ in range(6): |
| brain.decide("wait", FRAME) |
| assert len(brain.history) == 3 |
|
|
|
|
| def test_prompt_carries_the_goal_the_menu_and_the_done_option(): |
| brain = _ScriptedModel(['{"kind": "done"}']) |
| brain.decide("save the document", FRAME) |
| prompt = brain.prompts[0] |
| assert "save the document" in prompt |
| assert "2. Save (at" in prompt |
| assert '"kind": "done"' in prompt |
| assert "<number from the list>" in prompt |
|
|
|
|
| def test_text_only_brains_stop_when_there_is_nothing_to_click(): |
| class NeedsIndex(_ScriptedModel): |
| requires_index = True |
|
|
| with pytest.raises(GoalUnreachable, match="no clickable elements"): |
| NeedsIndex(['{"kind": "done"}']).decide("save", Observation(elements=())) |
|
|
|
|
| def test_a_right_answer_in_the_wrong_field_is_recovered(): |
| """Real 0.5B output: named the correct element but put it in "reason".""" |
| action = parse_decision('{"kind": "click", "index": 0, "reason": "Save"}', FRAME) |
| assert action.kind == "click" and action.label == "Save" |
| assert "recovered" in action.reason |
|
|
|
|
| def test_recovery_still_refuses_an_unresolvable_index(): |
| with pytest.raises(BrainError, match="not in the"): |
| parse_decision('{"kind": "click", "index": 42, "reason": "no idea"}', FRAME) |
|
|
|
|
| def test_recovery_is_not_ambiguous(): |
| |
| with pytest.raises(BrainError): |
| parse_decision('{"kind": "click", "index": 0, "reason": "Save or Cancel"}', FRAME) |
|
|
