File size: 8,890 Bytes
e2f048d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 | """A skill is only as trustworthy as the run that produced it."""
import pytest
from tiny_computer_agent import (
Action, AgentLoop, BrainError, DryRunExecutor, Element, Episode, FakeScreen, GoalUnreachable,
MockBrain, Observation, Policy, Skill, SkillBrain, SkillLibrary, SkillStep, Step,
record_skill, slug,
)
def save_button(label="Save", x=10, y=10):
return Element(label, x=x, y=y, width=60, height=20)
def screen_of(*labels, text=""):
return FakeScreen([tuple(save_button(l, x=10 + 80 * i) for i, l in enumerate(labels))], text=text)
def verified_click_episode(goal="click 'Save' to save the document", label="Save"):
"""A real, verified episode - produced by a real loop, not hand-built."""
loop = AgentLoop(screen=screen_of(label, text="Saved"),
brain=MockBrain([{"kind": "click", "label": label},
{"kind": "done", "reason": "clicked it"}]),
verify=lambda observation: "Saved" in observation.text)
return loop.run(goal)
def test_slugs_are_readable_and_stable():
assert slug("click 'Save' to save the document") == "save-document"
assert slug("open the File menu and choose Save As") == "open-file-menu-choose-save-as"
assert slug("") == "skill"
def test_an_unverified_run_is_never_written_down_as_a_skill():
"""The one thing this project does not do: turn a guess into a library others trust."""
episode = verified_click_episode()
episode.success = False
assert record_skill(episode) is None
empty = Episode(goal="do something", success=True)
assert record_skill(empty) is None, "no steps is not a skill"
def test_a_verified_run_becomes_a_reusable_skill():
skill = record_skill(verified_click_episode())
assert skill is not None
assert skill.verified and skill.name == "save-document"
assert len(skill.steps) == 1
assert skill.steps[0].action.kind == "click"
assert skill.steps[0].aimed_at == "Save"
assert skill.recorded, "a skill says when it was recorded"
def test_a_skill_stores_labels_and_never_coordinates():
"""Coordinates are the thing that stops working tomorrow."""
skill = record_skill(verified_click_episode())
action = skill.actions()[0]
assert action.label == "Save"
assert action.x is None and action.y is None
assert "Save" in skill.steps[0].describe()
def test_the_entry_condition_is_the_first_label_not_every_label():
skill = Skill(name="menu", goal="open the File menu and choose Save As", verified=True,
steps=(SkillStep(Action("click", label="File"), aimed_at="File"),
SkillStep(Action("click", label="Save As"), aimed_at="Save As")))
assert skill.preconditions() == ("File",), "Save As does not exist until the menu is open"
def test_a_library_matches_only_when_the_entry_label_is_on_screen():
library = SkillLibrary()
library.add(Skill(name="save", goal="click 'Save' to save the document", verified=True,
steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),)))
goal = "click 'Save' to save the document"
assert library.match(goal, Observation(elements=(save_button(),))) is not None
assert library.match(goal, Observation(elements=())) is None, "aim at nothing, match nothing"
assert library.match("format the hard disk", Observation(elements=(save_button(),))) is None
def test_a_library_round_trips_through_json(tmp_path):
library = SkillLibrary()
record_skill(verified_click_episode(), library=library)
path = library.save(tmp_path / "skills.json")
reloaded = SkillLibrary.load(path)
assert reloaded.names() == library.names()
skill = reloaded.get("save-document")
assert skill is not None and skill.steps[0].action.kind == "click"
assert skill.verified is True
assert SkillLibrary.load(tmp_path / "absent.json").names() == ()
assert reloaded.to_rows()[0]["needs"] == "Save"
def test_the_brain_regrounds_on_the_label_the_screen_shows_now():
library = SkillLibrary()
library.add(Skill(name="save", goal="click 'Save' to save the document", verified=True,
steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),)))
brain = SkillBrain(library)
action = brain.decide("click 'Save' to save the document",
Observation(elements=(save_button("Save."),)))
assert action.kind == "click"
assert action.label == "Save.", "the screen's spelling wins over the recorded one"
def test_a_skill_that_aimed_at_nothing_raises_instead_of_clicking():
library = SkillLibrary()
library.add(Skill(name="two", goal="save then confirm", verified=True,
steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),
SkillStep(Action("click", label="Confirm"), aimed_at="Confirm"))))
brain = SkillBrain(library)
goal = "save then confirm"
assert brain.decide(goal, Observation(elements=(save_button(),))).label == "Save"
with pytest.raises(BrainError, match="not on this screen"):
brain.decide(goal, Observation(elements=(save_button(),)))
def test_the_brain_says_done_when_the_skill_is_finished():
library = SkillLibrary()
library.add(Skill(name="one", goal="click 'Save' to save the document", verified=True,
steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),)))
brain = SkillBrain(library)
goal = "click 'Save' to save the document"
brain.decide(goal, Observation(elements=(save_button(),)))
finished = brain.decide(goal, Observation(elements=(save_button(),)))
assert finished.kind == "done" and "finished" in finished.reason
assert brain.progress == "1/1"
def test_the_brain_gives_up_instead_of_inventing_an_action():
brain = SkillBrain(SkillLibrary())
with pytest.raises(GoalUnreachable, match="no skill matches"):
brain.decide("format the hard disk", Observation(elements=(save_button(),)))
def test_the_brain_defers_to_a_fallback_then_reuses_the_choice():
library = SkillLibrary()
brain = SkillBrain(library, fallback=MockBrain([{"kind": "done", "reason": "fallback chose"}]))
action = brain.decide("click 'Save'", Observation(elements=(save_button(),)))
assert action.kind == "done" and "fallback" in action.reason
def test_a_skill_replays_through_a_real_loop_with_no_model():
library = SkillLibrary()
record_skill(verified_click_episode(), library=library)
goal = "click 'Save' to save the document"
loop = AgentLoop(screen=screen_of("Save", text="Saved"), brain=SkillBrain(library),
executor=DryRunExecutor(), policy=Policy(step_budget=3),
verify=lambda observation: "Saved" in observation.text)
episode = loop.run(goal)
assert episode.success, episode.render()
assert [s.action.label for s in episode.steps if s.action and s.action.kind == "click"] == ["Save"]
def test_a_skill_carries_the_check_that_made_it_trustworthy():
skill = Skill(name="s", goal="g", verified=True, verify_text="Saved",
steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),))
verify = skill.verifier()
assert verify is not None and verify(Observation(text="the file was Saved")) is True
assert verify(Observation(text="nothing happened")) is False
assert Skill(name="s", goal="g", steps=skill.steps).verifier() is None
def test_an_empty_skill_is_not_a_skill():
with pytest.raises(ValueError, match="no steps"):
Skill(name="empty", goal="nothing", steps=())
def test_a_skill_keeps_a_structural_check_across_a_round_trip(tmp_path):
"""A skill whose success is a dialog closing must survive being written down."""
episode = verified_click_episode()
library = SkillLibrary()
skill = record_skill(episode, library=library, verify_gone="Name")
assert skill is not None and skill.verified
assert skill.verification() == "'Name' goes away"
verify = skill.verifier()
assert verify is not None and callable(verify)
reloaded = SkillLibrary.load(library.save(tmp_path / "skills.json")).get(skill.name)
assert reloaded is not None
assert reloaded.verify_gone == "Name"
assert reloaded.verification() == "'Name' goes away"
def test_recording_refuses_a_claim_that_no_verifier_backed():
"""A brain reporting 'done' is a claim; only evidence makes a skill."""
from tiny_computer_agent import AgentLoop, MockBrain
loop = AgentLoop(screen=screen_of("Save", text=""), brain=MockBrain([{"kind": "done", "reason": "trust me"}]))
episode = loop.run("click 'Save' to save the document")
assert episode.success, "the loop takes the brain's word when there is no verifier"
assert record_skill(episode) is None, "but a skill library does not"
|