Publish tiny_computer_agent_app.py from marimo

tiny_computer_agent_app.py

@@ -17,10 +17,27 @@ with app.setup:
     import marimo as mo
     from huggingface_hub import HfApi
 
+    # The project's public API, imported once so no cell has to duplicate it.
+    from tiny_computer_agent import (
+        AgentLoop,
+        DryRunExecutor,
+        Element,
+        MockBrain,
+        PlannerBrain,
+        Policy,
+    )
+    from tiny_computer_agent.actions import Action
+    from tiny_computer_agent.brain import DecisionMenu
+    from tiny_computer_agent.demo import fake_desktop
+    from tiny_computer_agent.observe import ImageScreen
+    from tiny_computer_agent.perception import IndexedScreen, OcrDetector
+    from tiny_computer_agent.trace import Scenario, benchmark
+
+
 
 @app.cell(hide_code=True)
 def readme():
-    readme_md = '''
+    readme_md = r'''
 
     ---
     title: Tiny Computer Agent
@@ -46,7 +63,9 @@ def readme():
 
     > **Status 0.3.0: it drives a real desktop — on Linux and on Windows.** Screen capture,
     > an OCR element index with real coordinates, a real mouse click, and a *verified*
-    > outcome. The suite passes on a live Linux desktop and inside a real Windows Server
+    > outcome. A run can be saved as a portable *trace*, replayed offline and **scored**, so
+    > "which brain is right, and how often?" is a number in a table.
+    > The suite passes on a live Linux desktop and inside a real Windows Server
     > 2022 guest; what is not solved is model reliability — see
     > [measured results](#measured-results).
 
@@ -106,7 +125,7 @@ def readme():
     ```bash
     python -m pip install -e ".[desktop,ocr,dev]"
     pytest tests -q
-    # 123 passed, 2 skipped in 18.6s
+    # 151 passed, 2 skipped in 18.7s
     ```
 
     The two skips are the live-desktop tests. Give them a display and they run for real:
@@ -176,6 +195,37 @@ def readme():
       step 0 | click label='Save' reason="the reply names 'Save'" -> pixel (295, 284)
     ```
 
+    ### Scoring a brain, offline
+
+    "Which brain is right, and how often?" needs no desktop, no model and no network,
+    because a **trace** keeps every frame's element index and throws the pixels away. Record a
+    run once; replay it against any brain, as often as you like.
+
+    ```
+    $ python -m tiny_computer_agent --image example --trace runs/save.json --expected Save
+    ...
+    trace: wrote runs/save.json (2 frames)
+    score: correct   click 'Save' to save the document: expected 'Save', chose 'Save'
+    ```
+
+    A trace is JSON, so it diffs in review, commits next to the tests, and replays on a machine
+    with nothing installed. Scoring a whole suite is a table:
+
+    ```
+    scoreboard: 7/9 correct (78%)
+      planner (rule-based): 100%
+      first-item (baseline): 33%
+      oracle (always right): 100%
+      correct   save the document: expected 'Save', chose 'Save'
+      incorrect save the document: expected 'Save', chose 'Files' (chose 'Files', not 'Save')
+      ...
+    ```
+
+    The number is the brain's **decision** - what it chose when it had to commit. Whether the
+    harness *allowed* that decision to run is a separate axis: a click the policy refused is
+    still the choice it was, and it is flagged `(harness refused it)` rather than passed off as
+    a clean success. Keeping those two apart is the point.
+
     ### Which brain drove which run
 
     The harness and the brain are separable, and it matters which one a result came from.
@@ -207,7 +257,7 @@ def readme():
       `warning: no text was found on screen, so the index is 96 numbered grid cells and the
       agent is choosing blind` — instead of silently guessing.
 
-    **125 tests pass**, including real OCR over a real image and (opt-in) live-desktop tests
+    **153 tests pass**, including real OCR over a real image and (opt-in) live-desktop tests
     that click a real button.
 
     ## Verified on real Windows
@@ -249,7 +299,7 @@ def readme():
 
     None of the three failures in the first row is a harness defect: two are
     `ModuleNotFoundError: numpy` and the third is the guard bug documented below. With the
-    OCR engine installed, the same suite exits **0**.
+    OCR engine installed, the same suite exits **0**. (Those Windows counts predate `trace.py`; the harness suite has since grown to 153 tests.)
 
     **The emulated-OCR caveat.** onnxruntime requires AVX2, and QEMU's software emulation of
     AVX2 is roughly two orders of magnitude slower than native. A suite that finishes in 17
@@ -275,7 +325,7 @@ def readme():
 
     ## The test suite, in full
 
-    `pytest tests` collects **125 tests**. 123 run headless — no display, no network, no
+    `pytest tests` collects **153 tests**. 151 run headless — no display, no network, no
     weights. The last 2 are the opt-in live-desktop tests, skipped unless
     `TINY_COMPUTER_AGENT_LIVE_TEST=1`.
 
@@ -292,11 +342,12 @@ def readme():
     | `tests/test_grid_fallback.py` | 11 | the fallback index announces itself instead of guessing |
     | `tests/test_desktop.py` | 10 | display detection, the preflight checks, `doctor` |
     | `tests/test_cli.py` | 10 | the command line: `--doctor`, `--image`, `--live`, argument handling |
+    | `tests/test_trace.py` | 28 | traces: a run recorded with no pixels, replayed against another brain, and scored against ground truth |
     | `tests/test_live_desktop.py` | 2 | **needs a display, opt-in**: OCR finds a real button at the pixels it was drawn at, and a real click makes the app react |
 
     ```bash
-    pytest tests -q                                  # 123 passed, 2 skipped
-    TINY_COMPUTER_AGENT_LIVE_TEST=1 pytest tests -q  # 125 passed, on a desktop
+    pytest tests -q                                  # 151 passed, 2 skipped
+    TINY_COMPUTER_AGENT_LIVE_TEST=1 pytest tests -q  # 153 passed, on a desktop
     ```
 
     ## Install
@@ -305,7 +356,7 @@ def readme():
     pip install -e ".[ocr]"              # labels from pixels (RapidOCR: no system binary)
     pip install -e ".[desktop,ocr]"      # + real capture and real input
     pip install -e ".[vision,ocr]"       # + the tiny model brains
-    pytest                               # 125 tests, no display or network needed
+    pytest                               # 153 tests, no display or network needed
     ```
 
     ## Run it on your machine
@@ -382,7 +433,7 @@ def readme():
 
     # 2. inside the guest, after autologon
     #    vc_redist.x64.exe is required before onnxruntime will load
-    curl.exe -s -o C:	iny    c.exe <vc_redist-url> && C:	iny    c.exe /install /quiet /norestart
+    curl.exe -s -o C:\tiny\vc_redist.x64.exe <vc_redist-url> && C:\tiny\vc_redist.x64.exe /install /quiet /norestart
     ```
 
     Serve files to the guest over QEMU's user-mode networking (the host is `10.0.2.2`), which
@@ -406,6 +457,7 @@ def readme():
     | `loop.py` | `AgentLoop`, `Episode`, `Step`, the no-progress guard and the verifier |
     | `desktop.py` | Preflight checks, `doctor`, and `build_agent` in one call |
     | `demo.py` | The synthetic desktop used by tests and `--image example` |
+    | `trace.py` | Portable traces: record a run with no pixels, replay it offline, score the choice against ground truth |
 
     ## Why a tiny model can do this at all
 
@@ -438,7 +490,7 @@ def readme():
 
     **Works:** real capture, a real OCR index with coordinates that land on the right pixel,
     a real click an app reacts to, verified success, the closed vocabulary, the policy gates,
-    the no-progress guard, echo detection, the doctor, the CLI, 125 tests, a 256M VLM that
+    the no-progress guard, echo detection, the doctor, the CLI, portable traces that score a brain's choices offline, 153 tests, a 256M VLM that
     reads a button label from pixels on a clean screenshot — and the same suite passing inside
     a real Windows Server 2022 guest with no KVM (109 passed, exit 0, with the OCR-inference
     tests excluded as impractically slow under emulation).
@@ -459,7 +511,7 @@ def readme():
     - [ ] Make the OCR guard in `test_cli.py` skip, rather than fail, when no engine is installed
     - [ ] Make the VLM reliable on a live screen (it echoes the prompt instead of choosing)
     - [ ] Accessibility-tree detector (AT-SPI on Linux, UI Automation on Windows)
-    - [ ] Score episodes: replay a trace and count how often the choice was right
+    - [x] Score episodes: replay a trace and count how often the choice was right - see [scoring a brain, offline](#scoring-a-brain-offline)
     - [ ] marimo console: live screen preview, action log, pause / resume / step
 
     ## License
@@ -551,7 +603,7 @@ def repo_files():
     '''
 
     mo.md(f"packaging: {len(pyproject_toml) + len(license_txt) + len(gitignore_txt)} chars, "
-          f"v0.2.0 with ocr/desktop/vision extras")
+          f"v0.3.0 with ocr/desktop/vision extras")
     return gitignore_txt, license_txt, pyproject_toml
 
 
@@ -639,11 +691,6 @@ def publish_status(
 
 @app.cell
 def demo():
-    from tiny_computer_agent import AgentLoop, DryRunExecutor, PlannerBrain, Policy
-    from tiny_computer_agent.demo import fake_desktop
-    from tiny_computer_agent.observe import ImageScreen
-    from tiny_computer_agent.perception import IndexedScreen, OcrDetector
-
     # A real screenshot -> real OCR -> labelled, clickable elements. No display needed:
     # this is the same path --image takes, and it is what makes a tiny model viable.
     _detector = OcrDetector()
@@ -677,5 +724,90 @@ def demo():
     return
 
 
+@app.cell
+def scoreboard():
+    # The brains are separable, so they can be scored against the same tasks with no
+    # display, no OCR and no weights: a scenario is an element index plus the answer that
+    # is right. Record a run once; replay it against any brain, and count how often right.
+
+
+    class FirstItemBrain:
+        """A baseline with no model at all: click whatever is first in reading order."""
+
+        name = "first-item"
+
+        def decide(self, goal, observation):
+            menu = DecisionMenu(observation)
+            if not menu:
+                return Action("done", reason="nothing to click")
+            return Action("click", label=menu.items[0].label, reason="first item in reading order")
+
+
+    def _bar(label, x):
+        return Element(label, x=x, y=6, width=80, height=20)
+
+
+    def _button(label, x, y=270):
+        return Element(label, x=x, y=y, width=90, height=30)
+
+
+    score_scenarios = [
+        Scenario(
+            "click 'Save' to save the document",
+            frames=[
+                (_bar("Files", 0), _bar("Edit", 100), _bar("View", 200)),
+                (Element("Name", x=200, y=195, width=200, height=26), _button("Save", 200)),
+                (_bar("Files", 0), _button("Save", 200)),
+            ],
+            expected="Save", name="save the document",
+        ),
+        Scenario(
+            "click 'Files' to open the file manager",
+            frames=[(_bar("Files", 0), _bar("Edit", 100), _button("Save", 200))],
+            expected="Files", name="open the file manager",
+        ),
+        Scenario(
+            "click 'Cancel' to discard the change",
+            frames=[(_button("Save", 200), _button("Cancel", 400))],
+            expected="Cancel", name="discard the change",
+        ),
+    ]
+
+    scoreboard = benchmark(
+        score_scenarios,
+        {
+            "planner (rule-based)": lambda scenario: PlannerBrain(),
+            "first-item (baseline)": lambda scenario: FirstItemBrain(),
+            "oracle (always right)": lambda scenario: MockBrain(
+                [
+                    {"kind": "click", "label": scenario.expected},
+                    {"kind": "done", "reason": "clicked the answer the task names"},
+                ]
+            ),
+        },
+    )
+
+    mo.vstack(
+        [
+            mo.md(
+                f"**Score the choice, offline.** {len(score_scenarios)} tasks x "
+                f"{len(scoreboard.brains())} brains = **{scoreboard.total} decisions**, "
+                f"**{scoreboard.earned} of them right ({scoreboard.accuracy:.0%})** - read off "
+                f"saved element indexes, with no screen, no model and no network."
+            ),
+            mo.ui.table(scoreboard.to_rows(), selection=None),
+            mo.md(
+                "Accuracy per brain: "
+                + " · ".join(
+                    f"**{_brain}** {_accuracy:.0%}"
+                    for _brain, _accuracy in scoreboard.by_brain().items()
+                )
+            ),
+        ]
+    )
+
+    return
+
+
 if __name__ == "__main__":
     app.run()