"""The record must be checkable, and must say when it is not. A run's JSON can be edited by whoever would prefer a better result. These tests write a real log from a real episode, then do the three things a dishonest reader would try - edit a line, delete a line, reorder them - and require the verifier to name the entry at fault. """ import json from tiny_computer_agent import ( AgentLoop, AuditLog, Element, FakeScreen, MockBrain, observation_digest, verify_audit, ) def _screen(): return FakeScreen([(Element("Save", x=320, y=120, width=80, height=24),)]) def _brain(): return MockBrain([{"kind": "click", "label": "Save"}, {"kind": "done", "reason": "file saved"}]) def _run(tmp_path, name="audit.jsonl", **kwargs): log = AuditLog(tmp_path / name) loop = AgentLoop(screen=_screen(), brain=_brain(), audit=log, **kwargs) return loop.run("click 'Save' to save the document"), log # -- writing ------------------------------------------------------------------- # def test_a_run_writes_one_chained_entry_per_step_plus_a_closing_one(tmp_path): episode, log = _run(tmp_path) entries = log.entries() assert [e.kind for e in entries] == ["step", "step", "episode"] assert [e.index for e in entries] == [0, 1, 2] assert entries[0].prev == "", "the first entry follows nothing" assert entries[1].prev == entries[0].digest assert all(e.digest and len(e.digest) == 64 for e in entries) assert "click label='Save'" in entries[0].action def test_the_chain_verifies_after_a_real_run(tmp_path): _run(tmp_path) ok, detail = verify_audit(tmp_path / "audit.jsonl") assert ok, detail assert detail == "3 entries, chain intact" def test_the_closing_entry_carries_the_verdict_not_the_brains_word(tmp_path): episode, log = _run(tmp_path) assert episode.success closing = log.entries()[-1] assert closing.kind == "episode" assert closing.ok is True assert "success" in closing.outcome and episode.stop_reason in closing.outcome def test_render_counts_the_steps_and_the_chain(tmp_path): _, log = _run(tmp_path) line = log.render() assert "2 steps" in line assert "chain intact" in line def test_a_run_with_no_audit_writes_nothing(tmp_path): episode = AgentLoop(screen=_screen(), brain=_brain()).run("save") assert episode.success assert list(tmp_path.iterdir()) == [] def test_resuming_a_log_continues_its_chain(tmp_path): """A runner that is killed and restarted must not start a second, unrelated log.""" _run(tmp_path) episode, log = _run(tmp_path) # same path, fresh AuditLog assert episode.success entries = log.entries() assert [e.index for e in entries] == [0, 1, 2, 3, 4, 5] assert entries[3].prev == entries[2].digest assert verify_audit(tmp_path / "audit.jsonl")[0] is True # -- tampering ----------------------------------------------------------------- # def _rewrite(tmp_path, lines, name="audit.jsonl"): (tmp_path / name).write_text("\n".join(lines) + "\n", encoding="utf-8") def test_an_edited_entry_is_caught_and_named(tmp_path): _run(tmp_path) path = tmp_path / "audit.jsonl" lines = path.read_text(encoding="utf-8").splitlines() entry = json.loads(lines[1]) entry["outcome"] = "clicked without incident" # a nicer story lines[1] = json.dumps(entry, sort_keys=True, separators=(",", ":")) _rewrite(tmp_path, lines) ok, detail = verify_audit(path) assert ok is False assert "entry 2" in detail and "was edited" in detail def test_a_deleted_line_is_caught(tmp_path): _run(tmp_path) path = tmp_path / "audit.jsonl" lines = path.read_text(encoding="utf-8").splitlines() _rewrite(tmp_path, [lines[0], lines[2]]) ok, detail = verify_audit(path) assert ok is False assert "entry 2" in detail and "out of order" in detail def test_reordered_lines_are_caught(tmp_path): _run(tmp_path) path = tmp_path / "audit.jsonl" lines = path.read_text(encoding="utf-8").splitlines() _rewrite(tmp_path, [lines[1], lines[0], lines[2]]) ok, detail = verify_audit(path) assert ok is False assert "entry 1" in detail def test_a_missing_log_is_not_reported_as_fine(tmp_path): ok, detail = verify_audit(tmp_path / "nothing-here.jsonl") assert ok is False and "no audit log" in detail def test_an_empty_log_is_not_reported_as_fine(tmp_path): path = tmp_path / "empty.jsonl" path.write_text("") ok, detail = verify_audit(path) assert ok is False and "empty" in detail # -- the screen digest --------------------------------------------------------- # def test_the_digest_identifies_the_index_not_the_pixels(tmp_path): same = FakeScreen([(Element("Save", x=320, y=120, width=80, height=24),)]).capture() again = FakeScreen([(Element("Save", x=320, y=120, width=80, height=24),)]).capture() moved = FakeScreen([(Element("Save", x=400, y=120, width=80, height=24),)]).capture() renamed = FakeScreen([(Element("Cancel", x=320, y=120, width=80, height=24),)]).capture() assert observation_digest(same) == observation_digest(again) assert observation_digest(same) != observation_digest(moved) assert observation_digest(same) != observation_digest(renamed) assert len(observation_digest(same)) == 16