| """A skill is only as trustworthy as the run that produced it.""" |
|
|
| import pytest |
|
|
| from tiny_computer_agent import ( |
| Action, AgentLoop, BrainError, DryRunExecutor, Element, Episode, FakeScreen, GoalUnreachable, |
| MockBrain, Observation, Policy, Skill, SkillBrain, SkillLibrary, SkillStep, Step, |
| record_skill, slug, |
| ) |
|
|
|
|
| def save_button(label="Save", x=10, y=10): |
| return Element(label, x=x, y=y, width=60, height=20) |
|
|
|
|
| def screen_of(*labels, text=""): |
| return FakeScreen([tuple(save_button(l, x=10 + 80 * i) for i, l in enumerate(labels))], text=text) |
|
|
|
|
| def verified_click_episode(goal="click 'Save' to save the document", label="Save"): |
| """A real, verified episode - produced by a real loop, not hand-built.""" |
| loop = AgentLoop(screen=screen_of(label, text="Saved"), |
| brain=MockBrain([{"kind": "click", "label": label}, |
| {"kind": "done", "reason": "clicked it"}]), |
| verify=lambda observation: "Saved" in observation.text) |
| return loop.run(goal) |
|
|
|
|
| def test_slugs_are_readable_and_stable(): |
| assert slug("click 'Save' to save the document") == "save-document" |
| assert slug("open the File menu and choose Save As") == "open-file-menu-choose-save-as" |
| assert slug("") == "skill" |
|
|
|
|
| def test_an_unverified_run_is_never_written_down_as_a_skill(): |
| """The one thing this project does not do: turn a guess into a library others trust.""" |
| episode = verified_click_episode() |
| episode.success = False |
| assert record_skill(episode) is None |
|
|
| empty = Episode(goal="do something", success=True) |
| assert record_skill(empty) is None, "no steps is not a skill" |
|
|
|
|
| def test_a_verified_run_becomes_a_reusable_skill(): |
| skill = record_skill(verified_click_episode()) |
|
|
| assert skill is not None |
| assert skill.verified and skill.name == "save-document" |
| assert len(skill.steps) == 1 |
| assert skill.steps[0].action.kind == "click" |
| assert skill.steps[0].aimed_at == "Save" |
| assert skill.recorded, "a skill says when it was recorded" |
|
|
|
|
| def test_a_skill_stores_labels_and_never_coordinates(): |
| """Coordinates are the thing that stops working tomorrow.""" |
| skill = record_skill(verified_click_episode()) |
| action = skill.actions()[0] |
|
|
| assert action.label == "Save" |
| assert action.x is None and action.y is None |
| assert "Save" in skill.steps[0].describe() |
|
|
|
|
| def test_the_entry_condition_is_the_first_label_not_every_label(): |
| skill = Skill(name="menu", goal="open the File menu and choose Save As", verified=True, |
| steps=(SkillStep(Action("click", label="File"), aimed_at="File"), |
| SkillStep(Action("click", label="Save As"), aimed_at="Save As"))) |
|
|
| assert skill.preconditions() == ("File",), "Save As does not exist until the menu is open" |
|
|
|
|
| def test_a_library_matches_only_when_the_entry_label_is_on_screen(): |
| library = SkillLibrary() |
| library.add(Skill(name="save", goal="click 'Save' to save the document", verified=True, |
| steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),))) |
|
|
| goal = "click 'Save' to save the document" |
| assert library.match(goal, Observation(elements=(save_button(),))) is not None |
| assert library.match(goal, Observation(elements=())) is None, "aim at nothing, match nothing" |
| assert library.match("format the hard disk", Observation(elements=(save_button(),))) is None |
|
|
|
|
| def test_a_library_round_trips_through_json(tmp_path): |
| library = SkillLibrary() |
| record_skill(verified_click_episode(), library=library) |
| path = library.save(tmp_path / "skills.json") |
|
|
| reloaded = SkillLibrary.load(path) |
| assert reloaded.names() == library.names() |
| skill = reloaded.get("save-document") |
| assert skill is not None and skill.steps[0].action.kind == "click" |
| assert skill.verified is True |
| assert SkillLibrary.load(tmp_path / "absent.json").names() == () |
| assert reloaded.to_rows()[0]["needs"] == "Save" |
|
|
|
|
| def test_the_brain_regrounds_on_the_label_the_screen_shows_now(): |
| library = SkillLibrary() |
| library.add(Skill(name="save", goal="click 'Save' to save the document", verified=True, |
| steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),))) |
| brain = SkillBrain(library) |
|
|
| action = brain.decide("click 'Save' to save the document", |
| Observation(elements=(save_button("Save."),))) |
| assert action.kind == "click" |
| assert action.label == "Save.", "the screen's spelling wins over the recorded one" |
|
|
|
|
| def test_a_skill_that_aimed_at_nothing_raises_instead_of_clicking(): |
| library = SkillLibrary() |
| library.add(Skill(name="two", goal="save then confirm", verified=True, |
| steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"), |
| SkillStep(Action("click", label="Confirm"), aimed_at="Confirm")))) |
| brain = SkillBrain(library) |
| goal = "save then confirm" |
|
|
| assert brain.decide(goal, Observation(elements=(save_button(),))).label == "Save" |
| with pytest.raises(BrainError, match="not on this screen"): |
| brain.decide(goal, Observation(elements=(save_button(),))) |
|
|
|
|
| def test_the_brain_says_done_when_the_skill_is_finished(): |
| library = SkillLibrary() |
| library.add(Skill(name="one", goal="click 'Save' to save the document", verified=True, |
| steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),))) |
| brain = SkillBrain(library) |
| goal = "click 'Save' to save the document" |
| brain.decide(goal, Observation(elements=(save_button(),))) |
|
|
| finished = brain.decide(goal, Observation(elements=(save_button(),))) |
| assert finished.kind == "done" and "finished" in finished.reason |
| assert brain.progress == "1/1" |
|
|
|
|
| def test_the_brain_gives_up_instead_of_inventing_an_action(): |
| brain = SkillBrain(SkillLibrary()) |
| with pytest.raises(GoalUnreachable, match="no skill matches"): |
| brain.decide("format the hard disk", Observation(elements=(save_button(),))) |
|
|
|
|
| def test_the_brain_defers_to_a_fallback_then_reuses_the_choice(): |
| library = SkillLibrary() |
| brain = SkillBrain(library, fallback=MockBrain([{"kind": "done", "reason": "fallback chose"}])) |
| action = brain.decide("click 'Save'", Observation(elements=(save_button(),))) |
|
|
| assert action.kind == "done" and "fallback" in action.reason |
|
|
|
|
| def test_a_skill_replays_through_a_real_loop_with_no_model(): |
| library = SkillLibrary() |
| record_skill(verified_click_episode(), library=library) |
| goal = "click 'Save' to save the document" |
|
|
| loop = AgentLoop(screen=screen_of("Save", text="Saved"), brain=SkillBrain(library), |
| executor=DryRunExecutor(), policy=Policy(step_budget=3), |
| verify=lambda observation: "Saved" in observation.text) |
| episode = loop.run(goal) |
|
|
| assert episode.success, episode.render() |
| assert [s.action.label for s in episode.steps if s.action and s.action.kind == "click"] == ["Save"] |
|
|
|
|
| def test_a_skill_carries_the_check_that_made_it_trustworthy(): |
| skill = Skill(name="s", goal="g", verified=True, verify_text="Saved", |
| steps=(SkillStep(Action("click", label="Save"), aimed_at="Save"),)) |
| verify = skill.verifier() |
|
|
| assert verify is not None and verify(Observation(text="the file was Saved")) is True |
| assert verify(Observation(text="nothing happened")) is False |
| assert Skill(name="s", goal="g", steps=skill.steps).verifier() is None |
|
|
|
|
| def test_an_empty_skill_is_not_a_skill(): |
| with pytest.raises(ValueError, match="no steps"): |
| Skill(name="empty", goal="nothing", steps=()) |
|
|
|
|
| def test_a_skill_keeps_a_structural_check_across_a_round_trip(tmp_path): |
| """A skill whose success is a dialog closing must survive being written down.""" |
| episode = verified_click_episode() |
| library = SkillLibrary() |
| skill = record_skill(episode, library=library, verify_gone="Name") |
|
|
| assert skill is not None and skill.verified |
| assert skill.verification() == "'Name' goes away" |
| verify = skill.verifier() |
| assert verify is not None and callable(verify) |
|
|
| reloaded = SkillLibrary.load(library.save(tmp_path / "skills.json")).get(skill.name) |
| assert reloaded is not None |
| assert reloaded.verify_gone == "Name" |
| assert reloaded.verification() == "'Name' goes away" |
|
|
|
|
| def test_recording_refuses_a_claim_that_no_verifier_backed(): |
| """A brain reporting 'done' is a claim; only evidence makes a skill.""" |
| from tiny_computer_agent import AgentLoop, MockBrain |
|
|
| loop = AgentLoop(screen=screen_of("Save", text=""), brain=MockBrain([{"kind": "done", "reason": "trust me"}])) |
| episode = loop.run("click 'Save' to save the document") |
|
|
| assert episode.success, "the loop takes the brain's word when there is no verifier" |
| assert record_skill(episode) is None, "but a skill library does not" |
|
|
