Notebook: the operator layer, exercised in a cell
Browse files- tiny_computer_agent_app.py +78 -0
tiny_computer_agent_app.py
CHANGED
|
@@ -21,13 +21,18 @@ with app.setup:
|
|
| 21 |
# The project's public API, imported once so no cell has to duplicate it.
|
| 22 |
from tiny_computer_agent import (
|
| 23 |
AgentLoop,
|
|
|
|
| 24 |
DryRunExecutor,
|
| 25 |
Element,
|
|
|
|
| 26 |
GrabScreen,
|
| 27 |
MockBrain,
|
| 28 |
PlannerBrain,
|
| 29 |
Policy,
|
| 30 |
PyAutoGuiExecutor,
|
|
|
|
|
|
|
|
|
|
| 31 |
)
|
| 32 |
from tiny_computer_agent.actions import Action
|
| 33 |
from tiny_computer_agent.brain import DecisionMenu
|
|
@@ -1585,6 +1590,79 @@ def demo():
|
|
| 1585 |
return
|
| 1586 |
|
| 1587 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1588 |
@app.cell
|
| 1589 |
def scoreboard():
|
| 1590 |
# The brains are separable, so they can be scored against the same tasks with no
|
|
|
|
| 21 |
# The project's public API, imported once so no cell has to duplicate it.
|
| 22 |
from tiny_computer_agent import (
|
| 23 |
AgentLoop,
|
| 24 |
+
AuditLog,
|
| 25 |
DryRunExecutor,
|
| 26 |
Element,
|
| 27 |
+
FakeScreen,
|
| 28 |
GrabScreen,
|
| 29 |
MockBrain,
|
| 30 |
PlannerBrain,
|
| 31 |
Policy,
|
| 32 |
PyAutoGuiExecutor,
|
| 33 |
+
Watchdog,
|
| 34 |
+
gate_from_spec,
|
| 35 |
+
verify_audit,
|
| 36 |
)
|
| 37 |
from tiny_computer_agent.actions import Action
|
| 38 |
from tiny_computer_agent.brain import DecisionMenu
|
|
|
|
| 1590 |
return
|
| 1591 |
|
| 1592 |
|
| 1593 |
+
@app.cell
|
| 1594 |
+
def operator_layer():
|
| 1595 |
+
# The 2.0 operator layer, exercised rather than described: a gate that consents per action,
|
| 1596 |
+
# a watchdog that refuses to wait, and an audit log that can be checked afterwards.
|
| 1597 |
+
import json as _json
|
| 1598 |
+
import tempfile as _tempfile
|
| 1599 |
+
|
| 1600 |
+
_goal = "click 'Save' to save the document"
|
| 1601 |
+
|
| 1602 |
+
|
| 1603 |
+
def _room():
|
| 1604 |
+
"""A two-button screen, so the demo is about consent and evidence, not about OCR."""
|
| 1605 |
+
return FakeScreen([(Element("Save", x=320, y=120, width=80, height=24),
|
| 1606 |
+
Element("Cancel", x=440, y=120, width=80, height=24))])
|
| 1607 |
+
|
| 1608 |
+
|
| 1609 |
+
class _Hung(DryRunExecutor):
|
| 1610 |
+
"""An executor that never returns - the failure a step budget cannot catch."""
|
| 1611 |
+
|
| 1612 |
+
def execute(self, action, observation):
|
| 1613 |
+
import time
|
| 1614 |
+
|
| 1615 |
+
time.sleep(3)
|
| 1616 |
+
return super().execute(action, observation)
|
| 1617 |
+
|
| 1618 |
+
|
| 1619 |
+
_audit_path = Path(_tempfile.mkdtemp()) / "run.jsonl"
|
| 1620 |
+
_audit = AuditLog(_audit_path)
|
| 1621 |
+
_watchdog = Watchdog(step_timeout=30, wall_clock=120)
|
| 1622 |
+
|
| 1623 |
+
# 1. an ordinary episode, with a gate, a deadline and a record around it
|
| 1624 |
+
_episode = AgentLoop(
|
| 1625 |
+
screen=_room(), brain=PlannerBrain(), executor=DryRunExecutor(),
|
| 1626 |
+
policy=Policy(step_budget=3), gate=gate_from_spec("labels:Save"),
|
| 1627 |
+
audit=_audit, watchdog=_watchdog,
|
| 1628 |
+
).run(_goal)
|
| 1629 |
+
|
| 1630 |
+
# 2. the same goal behind a read-only gate: the operator's no is a step, not a crash
|
| 1631 |
+
_refused = AgentLoop(screen=_room(), brain=PlannerBrain(), executor=DryRunExecutor(),
|
| 1632 |
+
policy=Policy(step_budget=3),
|
| 1633 |
+
gate=gate_from_spec("read-only")).run(_goal)
|
| 1634 |
+
|
| 1635 |
+
# 3. a call that hangs: without the watchdog this cell would never finish
|
| 1636 |
+
_hung = AgentLoop(screen=_room(), brain=MockBrain([{"kind": "click", "label": "Save"}]),
|
| 1637 |
+
executor=_Hung(), watchdog=Watchdog(step_timeout=0.05)).run(_goal)
|
| 1638 |
+
|
| 1639 |
+
# 4. the record: check it, then tamper with a copy and let the verifier name the entry
|
| 1640 |
+
_lines = _audit_path.read_text().splitlines()
|
| 1641 |
+
_entry = _json.loads(_lines[0])
|
| 1642 |
+
_entry["outcome"] = "nothing happened at all" # a nicer story than the truth
|
| 1643 |
+
_tampered = _audit_path.with_name("tampered.jsonl")
|
| 1644 |
+
_tampered.write_text(_json.dumps(_entry, sort_keys=True, separators=(",", ":")) + "\n"
|
| 1645 |
+
+ "\n".join(_lines[1:]) + "\n")
|
| 1646 |
+
|
| 1647 |
+
operator_rows = [
|
| 1648 |
+
{"2.0 piece": "gate: labels:Save", "what happened": "let the whitelisted click through"},
|
| 1649 |
+
{"2.0 piece": "gate: read-only", "what happened": f"refused - {_refused.steps[-1].error}"},
|
| 1650 |
+
{"2.0 piece": "watchdog", "what happened": _hung.steps[-1].error},
|
| 1651 |
+
{"2.0 piece": "audit: intact", "what happened": verify_audit(_audit_path)[1]},
|
| 1652 |
+
{"2.0 piece": "audit: edited", "what happened": verify_audit(_tampered)[1]},
|
| 1653 |
+
]
|
| 1654 |
+
|
| 1655 |
+
operator_layer = mo.vstack([
|
| 1656 |
+
mo.md(f"**The 2.0 operator layer.** One goal - `{_goal}` - three ways: consented, refused, "
|
| 1657 |
+
f"and hung. {_watchdog.render()}"),
|
| 1658 |
+
mo.ui.table(operator_rows, selection=None),
|
| 1659 |
+
mo.md("```\n" + _episode.render() + "\n```"),
|
| 1660 |
+
])
|
| 1661 |
+
operator_layer
|
| 1662 |
+
|
| 1663 |
+
return
|
| 1664 |
+
|
| 1665 |
+
|
| 1666 |
@app.cell
|
| 1667 |
def scoreboard():
|
| 1668 |
# The brains are separable, so they can be scored against the same tasks with no
|