"""A skill is only as trustworthy as the run that produced it.""" import pytest from tiny_computer_agent import ( Action, AgentLoop, BrainError, DryRunExecutor, Element, Episode, FakeScreen, GoalUnreachable, MockBrain, Observation, Policy, Skill, SkillBrain, SkillLibrary, SkillStep, Step, record_skill, slug, ) def save_button(label="Save", x=10, y=10): return Element(label, x=x, y=y, width=60, height=20) def screen_of(*labels, text=""): return FakeScreen([tuple(save_button(l, x=10 + 80 * i) for i, l in enumerate(labels))], text=text) def verified_click_episode(goal="click 'Save' to save the document", label="Save"): """A real, verified episode - produced by a real loop, not hand-built.""" loop = AgentLoop(screen=screen_of(label, text="Saved"), brain=MockBrain([{"kind": "click", "label": label}, {"kind": "done", "reason": "clicked it"}]), verify=lambda observation: "Saved" in observation.text) return loop.run(goal) def test_slugs_are_readable_and_stable(): assert slug("click 'Save' to save the document") == "save-document" assert slug("open the File menu and choose Save As") == "open-file-menu-choose-save-as" assert slug("") == "skill" def test_an_unverified_run_is_never_written_down_as_a_skill(): """The one thing this project does not do: turn a guess into a library others trust.""" episode = verified_click_episode() episode.success = False assert record_skill(episode) is None empty = Episode(goal="do something", success=True) assert record_skill(empty) is None, "no steps is not a skill" def test_a_verified_run_becomes_a_reusable_skill(): skill = record_skill(verified_click_episode()) assert skill is not None assert skill.verified and skill.name == "save-document" assert len(skill.steps) == 1 assert skill.steps[0].action.kind == "click" assert skill.steps[0].aimed_at == "Save" assert skill.recorded, "a skill says when it was recorded" def test_a_skill_stores_labels_and_never_coordinates(): """Coordinates are the thing that stops working tomorrow.""" skill = record_skill(verified_click_episode()) action = skill.actions()[0] assert action.label == "Save" assert action.x is None and action.y is None assert "Save" in skill.steps[0].describe() def test_the_entry_condition_is_the_first_label_not_every_label(): skill = Skill(name="menu", goal="open the File menu and choose Save As", verified=True, steps=(SkillStep(Action("click", label="File"), aimed_at="File"), SkillStep(Action("click", label="Save As"), aimed_at="Save As"))) assert skill.preconditions() == ("File",), "Save As does not exist until the menu is open" def test_a_library_matches_only_when_the_entry_label_is_on_screen(): library = SkillLibrary() library.add(Skill(name="save", goal="click 'Save' to save the document", verified=True, steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),))) goal = "click 'Save' to save the document" assert library.match(goal, Observation(elements=(save_button(),))) is not None assert library.match(goal, Observation(elements=())) is None, "aim at nothing, match nothing" assert library.match("format the hard disk", Observation(elements=(save_button(),))) is None def test_a_library_round_trips_through_json(tmp_path): library = SkillLibrary() record_skill(verified_click_episode(), library=library) path = library.save(tmp_path / "skills.json") reloaded = SkillLibrary.load(path) assert reloaded.names() == library.names() skill = reloaded.get("save-document") assert skill is not None and skill.steps[0].action.kind == "click" assert skill.verified is True assert SkillLibrary.load(tmp_path / "absent.json").names() == () assert reloaded.to_rows()[0]["needs"] == "Save" def test_the_brain_regrounds_on_the_label_the_screen_shows_now(): library = SkillLibrary() library.add(Skill(name="save", goal="click 'Save' to save the document", verified=True, steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),))) brain = SkillBrain(library) action = brain.decide("click 'Save' to save the document", Observation(elements=(save_button("Save."),))) assert action.kind == "click" assert action.label == "Save.", "the screen's spelling wins over the recorded one" def test_a_skill_that_aimed_at_nothing_raises_instead_of_clicking(): library = SkillLibrary() library.add(Skill(name="two", goal="save then confirm", verified=True, steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"), SkillStep(Action("click", label="Confirm"), aimed_at="Confirm")))) brain = SkillBrain(library) goal = "save then confirm" assert brain.decide(goal, Observation(elements=(save_button(),))).label == "Save" with pytest.raises(BrainError, match="not on this screen"): brain.decide(goal, Observation(elements=(save_button(),))) def test_the_brain_says_done_when_the_skill_is_finished(): library = SkillLibrary() library.add(Skill(name="one", goal="click 'Save' to save the document", verified=True, steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),))) brain = SkillBrain(library) goal = "click 'Save' to save the document" brain.decide(goal, Observation(elements=(save_button(),))) finished = brain.decide(goal, Observation(elements=(save_button(),))) assert finished.kind == "done" and "finished" in finished.reason assert brain.progress == "1/1" def test_the_brain_gives_up_instead_of_inventing_an_action(): brain = SkillBrain(SkillLibrary()) with pytest.raises(GoalUnreachable, match="no skill matches"): brain.decide("format the hard disk", Observation(elements=(save_button(),))) def test_the_brain_defers_to_a_fallback_then_reuses_the_choice(): library = SkillLibrary() brain = SkillBrain(library, fallback=MockBrain([{"kind": "done", "reason": "fallback chose"}])) action = brain.decide("click 'Save'", Observation(elements=(save_button(),))) assert action.kind == "done" and "fallback" in action.reason def test_a_skill_replays_through_a_real_loop_with_no_model(): library = SkillLibrary() record_skill(verified_click_episode(), library=library) goal = "click 'Save' to save the document" loop = AgentLoop(screen=screen_of("Save", text="Saved"), brain=SkillBrain(library), executor=DryRunExecutor(), policy=Policy(step_budget=3), verify=lambda observation: "Saved" in observation.text) episode = loop.run(goal) assert episode.success, episode.render() assert [s.action.label for s in episode.steps if s.action and s.action.kind == "click"] == ["Save"] def test_a_skill_carries_the_check_that_made_it_trustworthy(): skill = Skill(name="s", goal="g", verified=True, verify_text="Saved", steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),)) verify = skill.verifier() assert verify is not None and verify(Observation(text="the file was Saved")) is True assert verify(Observation(text="nothing happened")) is False assert Skill(name="s", goal="g", steps=skill.steps).verifier() is None def test_an_empty_skill_is_not_a_skill(): with pytest.raises(ValueError, match="no steps"): Skill(name="empty", goal="nothing", steps=()) def test_a_skill_keeps_a_structural_check_across_a_round_trip(tmp_path): """A skill whose success is a dialog closing must survive being written down.""" episode = verified_click_episode() library = SkillLibrary() skill = record_skill(episode, library=library, verify_gone="Name") assert skill is not None and skill.verified assert skill.verification() == "'Name' goes away" verify = skill.verifier() assert verify is not None and callable(verify) reloaded = SkillLibrary.load(library.save(tmp_path / "skills.json")).get(skill.name) assert reloaded is not None assert reloaded.verify_gone == "Name" assert reloaded.verification() == "'Name' goes away" def test_recording_refuses_a_claim_that_no_verifier_backed(): """A brain reporting 'done' is a claim; only evidence makes a skill.""" from tiny_computer_agent import AgentLoop, MockBrain loop = AgentLoop(screen=screen_of("Save", text=""), brain=MockBrain([{"kind": "done", "reason": "trust me"}])) episode = loop.run("click 'Save' to save the document") assert episode.success, "the loop takes the brain's word when there is no verifier" assert record_skill(episode) is None, "but a skill library does not"