"""Success must be evidence, not a claim.""" import pytest from tiny_computer_agent import ( Action, AgentLoop, DryRunExecutor, Element, MockBrain, Observation, Policy, ) class ScriptedTextScreen: """Screen whose *text* changes, so a verifier has something to see.""" def __init__(self, texts): self.texts = list(texts) self.captures = 0 def capture(self) -> Observation: text = self.texts[min(self.captures, len(self.texts) - 1)] self.captures += 1 return Observation(index=self.captures - 1, elements=(Element("Save"),), text=text) WAITS = [{"kind": "wait", "seconds": 0.1}] * 4 # A script longer than the budget, so the budget - not a scripted "done" - ends the run. ENDLESS_WAITS = WAITS * 3 def run_with_budget(screen, brain, verify=None, budget=3): return AgentLoop(screen=screen, brain=brain, policy=Policy(step_budget=budget), verify=verify).run("save") def test_without_a_verifier_a_successful_action_can_look_like_a_failure(): """The bug this feature exists for: the click worked, the episode still failed.""" changing = ScriptedTextScreen(["one", "two", "three", "four"]) episode = run_with_budget(changing, MockBrain(ENDLESS_WAITS)) assert episode.success is False assert "budget" in episode.stop_reason def test_a_verifier_turns_evidence_into_success(): screen = ScriptedTextScreen(["Unsaved changes", "before", "Saved"]) episode = AgentLoop( screen=screen, brain=MockBrain(WAITS), verify=lambda obs: "saved" in obs.text.lower() ).run("save the document") assert episode.success is True assert episode.stop_reason == "verified after 2 action(s)" assert len(episode.steps) == 2 def test_a_verifier_that_never_matches_leaves_it_a_failure(): episode = AgentLoop( screen=ScriptedTextScreen(["nothing to see"]), brain=MockBrain(WAITS), verify=lambda obs: "saved" in obs.text.lower(), ).run("save") assert episode.success is False def test_a_broken_verifier_cannot_fake_a_success(): def explode(observation): raise RuntimeError("verifier bug") episode = run_with_budget( ScriptedTextScreen(["one", "two", "three", "four"]), MockBrain(ENDLESS_WAITS), verify=explode ) assert episode.success is False assert "budget" in episode.stop_reason def test_a_model_claiming_done_still_wins_when_there_is_no_verifier(): """Unchanged behaviour: with no verifier the brain's "done" is all we have.""" episode = AgentLoop( screen=ScriptedTextScreen(["x"]), brain=MockBrain([{"kind": "click", "label": "Save"}, {"kind": "done", "reason": "saved it"}]), ).run("save") assert episode.success is True and "saved it" in episode.stop_reason def test_verify_does_not_disturb_dry_run(): executor = DryRunExecutor() AgentLoop( screen=ScriptedTextScreen(["a", "b", "Saved"]), brain=MockBrain(WAITS), executor=executor, verify=lambda obs: "saved" in obs.text.lower(), ).run("save") assert all(a.kind == "wait" for a in executor.performed) def test_cli_verify_text_flag(capsys): from tiny_computer_agent import __main__ as cli # the demo screen's text says "Untitled document - unsaved changes" assert cli.main(["--script", '[{"kind": "wait", "seconds": 0.1}]', "--verify-text", "Unsaved", "--max-steps", "2"]) == 0 out = capsys.readouterr().out assert "verified after" in out # without it, the same run exhausts its budget and fails assert cli.main(["--script", '[{"kind": "wait", "seconds": 0.1}]', "--max-steps", "1"]) == 1