File size: 3,646 Bytes
8b9b8c8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | """Success must be evidence, not a claim."""
import pytest
from tiny_computer_agent import (
Action, AgentLoop, DryRunExecutor, Element, MockBrain, Observation, Policy,
)
class ScriptedTextScreen:
"""Screen whose *text* changes, so a verifier has something to see."""
def __init__(self, texts):
self.texts = list(texts)
self.captures = 0
def capture(self) -> Observation:
text = self.texts[min(self.captures, len(self.texts) - 1)]
self.captures += 1
return Observation(index=self.captures - 1, elements=(Element("Save"),), text=text)
WAITS = [{"kind": "wait", "seconds": 0.1}] * 4
# A script longer than the budget, so the budget - not a scripted "done" - ends the run.
ENDLESS_WAITS = WAITS * 3
def run_with_budget(screen, brain, verify=None, budget=3):
return AgentLoop(screen=screen, brain=brain, policy=Policy(step_budget=budget),
verify=verify).run("save")
def test_without_a_verifier_a_successful_action_can_look_like_a_failure():
"""The bug this feature exists for: the click worked, the episode still failed."""
changing = ScriptedTextScreen(["one", "two", "three", "four"])
episode = run_with_budget(changing, MockBrain(ENDLESS_WAITS))
assert episode.success is False
assert "budget" in episode.stop_reason
def test_a_verifier_turns_evidence_into_success():
screen = ScriptedTextScreen(["Unsaved changes", "before", "Saved"])
episode = AgentLoop(
screen=screen, brain=MockBrain(WAITS), verify=lambda obs: "saved" in obs.text.lower()
).run("save the document")
assert episode.success is True
assert episode.stop_reason == "verified after 2 action(s)"
assert len(episode.steps) == 2
def test_a_verifier_that_never_matches_leaves_it_a_failure():
episode = AgentLoop(
screen=ScriptedTextScreen(["nothing to see"]), brain=MockBrain(WAITS),
verify=lambda obs: "saved" in obs.text.lower(),
).run("save")
assert episode.success is False
def test_a_broken_verifier_cannot_fake_a_success():
def explode(observation):
raise RuntimeError("verifier bug")
episode = run_with_budget(
ScriptedTextScreen(["one", "two", "three", "four"]), MockBrain(ENDLESS_WAITS), verify=explode
)
assert episode.success is False
assert "budget" in episode.stop_reason
def test_a_model_claiming_done_still_wins_when_there_is_no_verifier():
"""Unchanged behaviour: with no verifier the brain's "done" is all we have."""
episode = AgentLoop(
screen=ScriptedTextScreen(["x"]),
brain=MockBrain([{"kind": "click", "label": "Save"}, {"kind": "done", "reason": "saved it"}]),
).run("save")
assert episode.success is True and "saved it" in episode.stop_reason
def test_verify_does_not_disturb_dry_run():
executor = DryRunExecutor()
AgentLoop(
screen=ScriptedTextScreen(["a", "b", "Saved"]), brain=MockBrain(WAITS), executor=executor,
verify=lambda obs: "saved" in obs.text.lower(),
).run("save")
assert all(a.kind == "wait" for a in executor.performed)
def test_cli_verify_text_flag(capsys):
from tiny_computer_agent import __main__ as cli
# the demo screen's text says "Untitled document - unsaved changes"
assert cli.main(["--script", '[{"kind": "wait", "seconds": 0.1}]',
"--verify-text", "Unsaved", "--max-steps", "2"]) == 0
out = capsys.readouterr().out
assert "verified after" in out
# without it, the same run exhausts its budget and fails
assert cli.main(["--script", '[{"kind": "wait", "seconds": 0.1}]', "--max-steps", "1"]) == 1
|