Publish tests/ from marimo

tests/test_act.py

@@ -0,0 +1,61 @@
+import pytest
+
+from tiny_computer_agent import (
+    Action, DryRunExecutor, Element, InputUnavailable, Observation,
+    PyAutoGuiExecutor, resolve_point,
+)
+
+FRAME = Observation(elements=(Element("Save", x=320, y=120, width=80, height=24),))
+
+
+def test_resolve_point_uses_pixels_when_given():
+    assert resolve_point(Action("click", x=1, y=2), FRAME) == (1, 2)
+
+
+def test_resolve_point_uses_the_element_index_for_labels():
+    assert resolve_point(Action("click", label="Save"), FRAME) == (360, 132)
+
+
+def test_resolve_point_is_none_for_unmatched_labels_and_other_verbs():
+    assert resolve_point(Action("click", label="Quit"), FRAME) is None
+    assert resolve_point(Action("type", text="hi"), FRAME) is None
+
+
+def test_dry_run_records_instead_of_acting():
+    executor = DryRunExecutor()
+    result = executor.execute(Action("click", label="Save"), FRAME)
+    assert result.ok
+    assert "(dry-run)" in result.note
+    assert "(360, 132)" in result.note
+    assert len(executor.performed) == 1
+
+
+def test_dry_run_fails_where_live_input_would_refuse():
+    """A dry run must never give false confidence about a doomed click."""
+    dry = DryRunExecutor().execute(Action("click", label="Quit"), FRAME)
+    live = PyAutoGuiExecutor().execute(Action("click", label="Quit"), FRAME)
+    assert dry.ok is False and live.ok is False
+    assert "absent from the element index" in dry.note
+
+
+def test_live_executor_refuses_to_guess_pixels():
+    """The dangerous path: a label with no element behind it must never click."""
+    result = PyAutoGuiExecutor().execute(Action("click", label="Quit"), FRAME)
+    assert result.ok is False
+    assert "refusing to guess pixels" in result.note
+
+
+def test_live_executor_treats_done_as_a_no_op():
+    result = PyAutoGuiExecutor().execute(Action("done", reason="finished"))
+    assert result.ok and "goal reported met" in result.note
+
+
+def test_live_executor_needs_pyautogui_installed():
+    """Real input on a machine without pyautogui fails loudly, not silently."""
+    try:
+        import pyautogui  # noqa: F401
+    except ImportError:
+        with pytest.raises(InputUnavailable, match="pyautogui"):
+            PyAutoGuiExecutor().execute(Action("type", text="hello"))
+    else:
+        pytest.skip("pyautogui is installed here")

tests/test_actions.py

@@ -0,0 +1,54 @@
+import pytest
+
+from tiny_computer_agent import ACTION_KINDS, ACTION_MENU, Action, ActionError, parse_action
+
+
+def test_click_by_label():
+    action = Action("click", label="Save")
+    assert action.kind == "click"
+    assert "label='Save'" in action.describe()
+
+
+def test_click_by_pixels():
+    assert Action("click", x=10, y=20).x == 10
+
+
+@pytest.mark.parametrize(
+    "kwargs, message",
+    [
+        ({"kind": "fly"}, "unknown action"),
+        ({"kind": "click"}, "click needs a label"),
+        ({"kind": "click", "x": -1, "y": 5}, "non-negative"),
+        ({"kind": "type", "text": ""}, "type needs non-empty text"),
+        ({"kind": "key"}, "key needs a chord"),
+        ({"kind": "scroll", "amount": 0}, "non-zero amount"),
+        ({"kind": "wait", "seconds": 0}, "positive number of seconds"),
+    ],
+)
+def test_validation_rejects_bad_actions(kwargs, message):
+    with pytest.raises(ActionError, match=message):
+        Action(**kwargs)
+
+    # the same rule must hold for anything a model sends back
+    with pytest.raises(ActionError):
+        parse_action(kwargs)
+
+
+def test_parse_action_ignores_chatty_extra_keys():
+    action = parse_action({"kind": "type", "text": "hello", "confidence": 0.9, "thought": "..."})
+    assert action.text == "hello"
+
+
+def test_parse_action_rejects_a_non_mapping():
+    with pytest.raises(ActionError, match="expected a mapping"):
+        parse_action(["click", "Save"])
+
+
+def test_menu_documents_every_verb():
+    for kind in ACTION_KINDS:
+        assert f'"kind": "{kind}"' in ACTION_MENU
+
+
+def test_round_trip_to_dict():
+    action = Action("scroll", amount=-3, reason="more content below")
+    assert parse_action(action.to_dict()).amount == -3

tests/test_cli.py

@@ -0,0 +1,33 @@
+import pytest
+
+from tiny_computer_agent import __main__ as cli
+
+
+def test_demo_runs_dry_run_by_default(capsys):
+    assert cli.main([]) == 0
+    out = capsys.readouterr().out
+    assert "dry-run" in out
+    assert "(dry-run) would click" in out
+    assert "result: success" in out
+
+
+def test_real_input_needs_the_allow_env_var(monkeypatch, capsys):
+    monkeypatch.delenv(cli.ALLOW_REAL_INPUT_ENV, raising=False)
+    assert cli.main(["--real-input"]) == 2
+    assert "refusing --real-input" in capsys.readouterr().err
+
+
+def test_bad_script_json_is_reported(capsys):
+    assert cli.main(["--script", "{not json"]) == 2
+    assert "not valid JSON" in capsys.readouterr().err
+
+
+def test_script_replay(capsys):
+    assert cli.main(["--script", '[{"kind": "done", "reason": "nothing to do"}]']) == 0
+    assert "nothing to do" in capsys.readouterr().out
+
+
+def test_failure_exit_code(capsys):
+    # the planner gives up when the target never appears, so the CLI exits non-zero
+    assert cli.main(["click 'Nonexistent button'"]) == 1
+    assert "gave up" in capsys.readouterr().out

tests/test_loop.py

@@ -0,0 +1,135 @@
+import pytest
+
+from tiny_computer_agent import (
+    Action, AgentLoop, DryRunExecutor, Element, Episode, FakeScreen,
+    GoalUnreachable, MockBrain, PlannerBrain, Policy,
+)
+from tiny_computer_agent.__main__ import demo_screen
+
+
+def test_scripted_episode_succeeds_and_records_every_step():
+    screen = FakeScreen([(Element("Save", x=320, y=120, width=80, height=24),)])
+    brain = MockBrain([
+        {"kind": "type", "text": "report"},
+        {"kind": "key", "keys": "ctrl+s"},
+        {"kind": "done", "reason": "file saved"},
+    ])
+    episode = AgentLoop(screen=screen, brain=brain).run("save the report")
+
+    assert episode.success
+    assert [s.action.kind for s in episode.steps] == ["type", "key", "done"]
+    assert "file saved" in episode.stop_reason
+    assert len(episode.to_rows()) == 3
+
+
+def test_planner_solves_a_real_click_task_with_no_model():
+    episode = AgentLoop(screen=demo_screen(), brain=PlannerBrain()).run(
+        "click 'Save' to save the document"
+    )
+    assert episode.success
+    clicks = [s.action for s in episode.steps if s.action.kind == "click"]
+    assert [c.label for c in clicks] == ["Save"]
+
+
+def test_screen_is_reread_after_every_executed_action():
+    """The verify half of the loop: never decide twice on the same frame."""
+    screen = FakeScreen([(Element("Save"),)])
+    brain = MockBrain([{"kind": "click", "label": "Save"}, {"kind": "click", "label": "Save"}])
+    episode = AgentLoop(screen=screen, brain=brain).run("click Save")
+
+    executed = [s for s in episode.steps if s.result is not None and s.action.kind != "done"]
+    assert len(executed) == 2
+    assert screen.captures == len(executed) + 1  # one initial read, one per action
+    assert episode.steps[0].observation.index == 0
+    assert episode.steps[1].observation.index == 1
+
+
+def test_step_budget_is_enforced():
+    screen = FakeScreen([(Element("Save"),)])
+    brain = MockBrain([{"kind": "click", "label": "Save"}] * 10)
+    episode = AgentLoop(
+        screen=screen, brain=brain, policy=Policy(step_budget=3)
+    ).run("click Save")
+
+    assert episode.success is False
+    assert len(episode.steps) == 3
+    assert "budget of 3" in episode.stop_reason
+
+
+def test_policy_refusal_stops_the_run_and_nothing_executes():
+    executor = DryRunExecutor()
+    episode = AgentLoop(
+        screen=FakeScreen([(Element("Terminal"),)]),
+        brain=MockBrain([{"kind": "type", "text": "sudo rm -rf /"}]),
+        executor=executor,
+    ).run("clean up")
+
+    assert episode.success is False
+    assert "policy refusal" in episode.stop_reason
+    assert episode.steps[-1].error.startswith("refused:")
+    assert executor.performed == []
+
+
+def test_kill_switch_aborts_before_the_first_action(tmp_path):
+    marker = tmp_path / "STOP"
+    marker.write_text("stop")
+    episode = AgentLoop(
+        screen=FakeScreen([(Element("Save"),)]),
+        brain=MockBrain([{"kind": "click", "label": "Save"}]),
+        policy=Policy(kill_switch_path=str(marker)),
+    ).run("click Save")
+
+    assert episode.steps == []
+    assert episode.stop_reason == "kill switch engaged"
+
+
+def test_a_broken_brain_stops_the_run_instead_of_crashing_it():
+    class Exploding:
+        name = "exploding"
+
+        def decide(self, goal, observation):
+            raise RuntimeError("model died")
+
+    episode = AgentLoop(screen=FakeScreen([(Element("Save"),)]), brain=Exploding()).run("save")
+
+    assert episode.success is False
+    assert episode.stop_reason == "brain error"
+    assert "RuntimeError: model died" in episode.steps[-1].error
+
+
+def test_failed_action_ends_the_episode():
+    episode = AgentLoop(
+        screen=FakeScreen([(Element("Save"),)]),
+        brain=MockBrain([{"kind": "click", "label": "Nowhere"}]),
+    ).run("click Nowhere")
+
+    assert episode.success is False
+    assert "action failed" in episode.stop_reason
+
+
+def test_on_step_callback_sees_every_step_and_render_mentions_the_goal():
+    seen = []
+    episode = AgentLoop(
+        screen=demo_screen(), brain=PlannerBrain(), on_step=seen.append
+    ).run("press Save")
+
+    assert len(seen) == len(episode.steps)
+    assert "press Save" in episode.render()
+    assert "success" in episode.render()
+
+
+def test_a_brain_that_gives_up_is_a_failure_not_a_success():
+    """The harness owns success: "I give up" must never read as "goal met"."""
+    episode = AgentLoop(
+        screen=FakeScreen([(Element("Files"),)]),
+        brain=PlannerBrain(max_waits=0),
+    ).run("click 'Save'")
+
+    assert episode.success is False
+    assert episode.stop_reason.startswith("brain gave up:")
+    assert episode.steps[-1].error.startswith("gave up:")
+
+
+def test_planner_raises_rather_than_inventing_a_target():
+    with pytest.raises(GoalUnreachable, match="no element name"):
+        PlannerBrain().decide("do the thing", FakeScreen([(Element("Save"),)]).capture())

tests/test_observe.py

@@ -0,0 +1,54 @@
+import pytest
+
+from tiny_computer_agent import Element, FakeScreen, Observation
+
+
+def test_element_center_is_the_middle_of_the_box():
+    assert Element("Save", x=320, y=120, width=80, height=24).center == (360, 132)
+
+
+def test_element_center_falls_back_to_corner_without_size():
+    assert Element("dot", x=5, y=7).center == (5, 7)
+
+
+def test_element_rejects_empty_label():
+    with pytest.raises(ValueError, match="non-empty"):
+        Element("   ")
+
+
+def test_element_rejects_negative_geometry():
+    with pytest.raises(ValueError, match="non-negative"):
+        Element("Save", x=-5)
+
+
+def test_fake_screen_advances_then_holds_the_last_frame():
+    screen = FakeScreen([(Element("A"),), (Element("B"),)])
+    assert [e.label for f in range(4) for e in screen.capture().elements] == ["A", "B", "B", "B"]
+    assert screen.captures == 4
+
+
+def test_find_prefers_exact_match_over_substring():
+    observation = Observation(elements=(
+        Element("Save as"), Element("Save"), Element("Save all"),
+    ))
+    assert observation.find("Save").label == "Save"       # exact wins
+    assert observation.find("save as").label == "Save as"  # case-insensitive
+    assert observation.find("ave a").label == "Save as"    # substring fallback
+    assert observation.find("Quit") is None
+
+
+def test_summary_lists_elements_and_truncates():
+    observation = Observation(
+        index=2,
+        elements=tuple(Element(f"button {i}") for i in range(5)),
+        text="  Untitled\n  document ",
+    )
+    summary = observation.summary(max_elements=3)
+    assert "step 2" in summary
+    assert "Untitled document" in summary          # whitespace collapsed
+    assert "button 0" in summary
+    assert "and 2 more" in summary
+
+
+def test_summary_admits_when_there_is_no_index():
+    assert "no element index" in Observation().summary()

tests/test_safety.py

@@ -0,0 +1,55 @@
+import pytest
+
+from tiny_computer_agent import (
+    Action, Element, Observation, Policy, PolicyViolation,
+)
+
+FRAME = Observation(elements=(Element("Terminal"), Element("Save")))
+
+
+def test_allows_an_ordinary_click():
+    Policy().check(Action("click", label="Save"), FRAME)
+
+
+def test_refuses_a_denied_key_chord():
+    with pytest.raises(PolicyViolation, match="deny list"):
+        Policy().check(Action("key", keys="ctrl+alt+t"))
+
+
+@pytest.mark.parametrize("text", ["sudo apt install foo", "rm -rf /", "my password is hunter2"])
+def test_refuses_denied_text(text):
+    with pytest.raises(PolicyViolation, match="deny list"):
+        Policy().check(Action("type", text=text))
+
+
+def test_refuses_clicking_a_denied_widget():
+    with pytest.raises(PolicyViolation, match="Terminal"):
+        Policy().check(Action("click", label="Terminal"), FRAME)
+
+
+def test_deny_label_is_case_insensitive():
+    with pytest.raises(PolicyViolation):
+        Policy().check(Action("click", label="terminal"), FRAME)
+
+
+def test_kill_switch_is_off_until_the_file_exists(tmp_path):
+    marker = tmp_path / "STOP"
+    policy = Policy(kill_switch_path=str(marker))
+    assert policy.should_stop() is False
+    marker.write_text("stop")
+    assert policy.should_stop() is True
+
+
+def test_no_kill_switch_path_means_never_stops():
+    assert Policy().should_stop() is False
+
+
+def test_rejects_a_useless_budget():
+    with pytest.raises(ValueError, match="step_budget"):
+        Policy(step_budget=0)
+
+
+def test_deny_lists_are_configurable():
+    lax = Policy(deny_text_patterns=(), deny_labels=(), deny_keys=())
+    lax.check(Action("type", text="sudo rm -rf /"))
+    lax.check(Action("click", label="Terminal"), FRAME)