| """Success must be evidence, not a claim.""" |
|
|
| import pytest |
|
|
| from tiny_computer_agent import ( |
| Action, AgentLoop, DryRunExecutor, Element, MockBrain, Observation, Policy, |
| ) |
|
|
|
|
| class ScriptedTextScreen: |
| """Screen whose *text* changes, so a verifier has something to see.""" |
|
|
| def __init__(self, texts): |
| self.texts = list(texts) |
| self.captures = 0 |
|
|
| def capture(self) -> Observation: |
| text = self.texts[min(self.captures, len(self.texts) - 1)] |
| self.captures += 1 |
| return Observation(index=self.captures - 1, elements=(Element("Save"),), text=text) |
|
|
|
|
| WAITS = [{"kind": "wait", "seconds": 0.1}] * 4 |
| |
| ENDLESS_WAITS = WAITS * 3 |
|
|
|
|
| def run_with_budget(screen, brain, verify=None, budget=3): |
| return AgentLoop(screen=screen, brain=brain, policy=Policy(step_budget=budget), |
| verify=verify).run("save") |
|
|
|
|
| def test_without_a_verifier_a_successful_action_can_look_like_a_failure(): |
| """The bug this feature exists for: the click worked, the episode still failed.""" |
| changing = ScriptedTextScreen(["one", "two", "three", "four"]) |
| episode = run_with_budget(changing, MockBrain(ENDLESS_WAITS)) |
| assert episode.success is False |
| assert "budget" in episode.stop_reason |
|
|
|
|
| def test_a_verifier_turns_evidence_into_success(): |
| screen = ScriptedTextScreen(["Unsaved changes", "before", "Saved"]) |
| episode = AgentLoop( |
| screen=screen, brain=MockBrain(WAITS), verify=lambda obs: "saved" in obs.text.lower() |
| ).run("save the document") |
|
|
| assert episode.success is True |
| assert episode.stop_reason == "verified after 2 action(s)" |
| assert len(episode.steps) == 2 |
|
|
|
|
| def test_a_verifier_that_never_matches_leaves_it_a_failure(): |
| episode = AgentLoop( |
| screen=ScriptedTextScreen(["nothing to see"]), brain=MockBrain(WAITS), |
| verify=lambda obs: "saved" in obs.text.lower(), |
| ).run("save") |
| assert episode.success is False |
|
|
|
|
| def test_a_broken_verifier_cannot_fake_a_success(): |
| def explode(observation): |
| raise RuntimeError("verifier bug") |
|
|
| episode = run_with_budget( |
| ScriptedTextScreen(["one", "two", "three", "four"]), MockBrain(ENDLESS_WAITS), verify=explode |
| ) |
| assert episode.success is False |
| assert "budget" in episode.stop_reason |
|
|
|
|
| def test_a_model_claiming_done_still_wins_when_there_is_no_verifier(): |
| """Unchanged behaviour: with no verifier the brain's "done" is all we have.""" |
| episode = AgentLoop( |
| screen=ScriptedTextScreen(["x"]), |
| brain=MockBrain([{"kind": "click", "label": "Save"}, {"kind": "done", "reason": "saved it"}]), |
| ).run("save") |
| assert episode.success is True and "saved it" in episode.stop_reason |
|
|
|
|
| def test_verify_does_not_disturb_dry_run(): |
| executor = DryRunExecutor() |
| AgentLoop( |
| screen=ScriptedTextScreen(["a", "b", "Saved"]), brain=MockBrain(WAITS), executor=executor, |
| verify=lambda obs: "saved" in obs.text.lower(), |
| ).run("save") |
| assert all(a.kind == "wait" for a in executor.performed) |
|
|
|
|
| def test_cli_verify_text_flag(capsys): |
| from tiny_computer_agent import __main__ as cli |
|
|
| |
| assert cli.main(["--script", '[{"kind": "wait", "seconds": 0.1}]', |
| "--verify-text", "Unsaved", "--max-steps", "2"]) == 0 |
| out = capsys.readouterr().out |
| assert "verified after" in out |
|
|
| |
| assert cli.main(["--script", '[{"kind": "wait", "seconds": 0.1}]', "--max-steps", "1"]) == 1 |
|
|
