Publish tiny_computer_agent_app.py from marimo

tiny_computer_agent_app.py

@@ -1,7 +1,7 @@
 # /// script
 # requires-python = ">=3.13"
 # dependencies = [
-#     "pillow>=10",
+#     "pillow==13.0.0",
 #     "rapidocr-onnxruntime==1.4.4",
 # ]
 # ///
@@ -23,9 +23,11 @@ with app.setup:
         AgentLoop,
         DryRunExecutor,
         Element,
+        GrabScreen,
         MockBrain,
         PlannerBrain,
         Policy,
+        PyAutoGuiExecutor,
     )
     from tiny_computer_agent.actions import Action
     from tiny_computer_agent.brain import DecisionMenu
@@ -64,7 +66,11 @@ def readme():
 
     > **Status 0.3.0: it drives a real desktop — on Linux and on Windows.** Screen capture,
     > an OCR element index with real coordinates, a real mouse click, and a *verified*
-    > outcome. **All 68 capabilities have a real screenshot** in the [capability gallery](#capability-gallery), and a run can be saved as a portable *trace*, replayed offline and **scored**, so
+    > outcome. **All 73 capabilities have a real screenshot** in the [capability gallery](#capability-gallery).
+    > The loop can be driven from a [marimo console](#the-marimo-console) with a live preview, an
+    > action log and pause / resume / step; elements can come from the
+    > [accessibility tree](#what-the-machine-already-knows-the-accessibility-tree) instead of OCR;
+    > and a run can be saved as a portable *trace*, replayed offline and **scored**, so
     > "which brain is right, and how often?" is a number in a table.
     > The suite passes on a live Linux desktop and inside a real Windows Server
     > 2022 guest; what is not solved is model reliability — see
@@ -126,7 +132,7 @@ def readme():
     ```bash
     python -m pip install -e ".[desktop,ocr,dev]"
     pytest tests -q
-    # 162 passed, 2 skipped in 18.7s
+    # 217 passed, 2 skipped in 18.7s
     ```
 
     The two skips are the live-desktop tests. Give them a display and they run for real:
@@ -227,6 +233,46 @@ def readme():
     still the choice it was, and it is flagged `(harness refused it)` rather than passed off as
     a clean success. Keeping those two apart is the point.
 
+    ### Asking a tiny model in four ways (measured)
+
+    The card used to say the 256M VLM "echoes the prompt instead of choosing". That is
+    reproducible - and so is the fix. A live GTK window, a menu of four exact widgets
+    (Files, Terminal, Save, Cancel), and the goal `click 'Save' to save the document`:
+
+    ```
+    mode     echoed  seconds  outcome
+    json     no          8.9  BrainError: the reply mentions 4 menu entries, so it looks like an echo
+    choice   no          1.9  BrainError: the constrained reply was not a menu number: '1000'
+    scan     no          1.4  click label='Files'   (the model said yes to the first candidate)
+    rank     no          9.4  click label='Files'   (highest yes/no log-odds margin, +0.94)
+    ```
+
+    Four ways to ask, all of them the same weights:
+
+    | mode | how the model is asked | answer space |
+    |---|---|---|
+    | `json` | reply with JSON, parsed leniently | the whole vocabulary |
+    | `choice` | one masked generation for a menu number | a handful of digits |
+    | `scan` | one masked yes/no question per item, first yes wins | two tokens |
+    | `rank` | the same questions, ranked by the yes/no log-odds margin | two tokens |
+
+    What the numbers actually say:
+
+    - **The echo is fixed, structurally.** In `json` mode the model hands the prompt
+      back (`"The goal is to click 'Save' to save the document..."`) and the harness
+      refuses it. In the three masked modes the reply *cannot* be the prompt, because
+      every one of its tokens is masked out of the logits. The failure stopped being
+      *detected* and started being *not representable*.
+    - **It is about six times faster** - 8.9 s of free text becomes 1.4 s for a
+      one-token answer - because the model is no longer asked to write.
+    - **Accuracy is not fixed, and the numbers say why.** The model answers now, and
+      answers *wrongly*: it accepts `Files` for a goal that names `Save`, even when the
+      answer space is two tokens and each candidate is judged on its own. A 256M model
+      cannot do this task. That is a finding about the weights, not about the harness -
+      which is why the rule-based planner is still the default brain.
+
+    Reproduce it with `python measure_vlm.py --screen live --detector best --open-gtk`.
+
     ### Which brain drove which run
 
     The harness and the brain are separable, and it matters which one a result came from.
@@ -258,7 +304,7 @@ def readme():
       `warning: no text was found on screen, so the index is 96 numbered grid cells and the
       agent is choosing blind` — instead of silently guessing.
 
-    **164 tests pass**, including real OCR over a real image and (opt-in) live-desktop tests
+    **219 tests pass**, including real OCR over a real image and (opt-in) live-desktop tests
     that click a real button.
 
     ## Verified on real Windows
@@ -300,7 +346,7 @@ def readme():
 
     None of the three failures in the first row is a harness defect: two are
     `ModuleNotFoundError: numpy` and the third is the guard bug documented below. With the
-    OCR engine installed, the same suite exits **0**. (Those Windows counts predate `trace.py`; the harness suite has since grown to 164 tests.)
+    OCR engine installed, the same suite exits **0**. (Those Windows counts predate `trace.py`; the harness suite has since grown to 219 tests.)
 
     **The emulated-OCR caveat.** onnxruntime requires AVX2, and QEMU's software emulation of
     AVX2 is roughly two orders of magnitude slower than native. A suite that finishes in 17
@@ -326,7 +372,7 @@ def readme():
 
     ## The test suite, in full
 
-    `pytest tests` collects **164 tests**. 162 run headless — no display, no network, no
+    `pytest tests` collects **219 tests**. 217 run headless — no display, no network, no
     weights. The last 2 are the opt-in live-desktop tests, skipped unless
     `TINY_COMPUTER_AGENT_LIVE_TEST=1`.
 
@@ -346,11 +392,15 @@ def readme():
     | `tests/test_trace.py` | 28 | traces: a run recorded with no pixels, replayed against another brain, and scored against ground truth |
     | `tests/test_models.py` | 7 | the model backends: lazy loading, device choice, honest refusals |
     | `tests/test_api_surface.py` | 4 | the public API is a promise: everything advertised is importable, and nothing public is left out |
+    | `tests/test_decoding.py` | 24 | the mask, the answer space, the retry ladder, the vote - and that an echo is refused |
+    | `tests/test_accessibility.py` | 16 | role filtering, the coordinate contract, the fallback chain, and real AT-SPI |
+    | `tests/test_stepwise.py` | 11 | begin / step / pause / resume, and that it equals the batch loop |
+    | `tests/test_ocr_guard.py` | 4 | the OCR guard really skips with no engine - the guard that never fired |
     | `tests/test_live_desktop.py` | 2 | **needs a display, opt-in**: OCR finds a real button at the pixels it was drawn at, and a real click makes the app react |
 
     ```bash
-    pytest tests -q                                  # 162 passed, 2 skipped
-    TINY_COMPUTER_AGENT_LIVE_TEST=1 pytest tests -q  # 164 passed, on a desktop
+    pytest tests -q                                  # 217 passed, 2 skipped
+    TINY_COMPUTER_AGENT_LIVE_TEST=1 pytest tests -q  # 219 passed, on a desktop
     ```
 
     ## Install
@@ -359,7 +409,15 @@ def readme():
     pip install -e ".[ocr]"              # labels from pixels (RapidOCR: no system binary)
     pip install -e ".[desktop,ocr]"      # + real capture and real input
     pip install -e ".[vision,ocr]"       # + the tiny model brains
-    pytest                               # 164 tests, no display or network needed
+    pytest                               # 219 tests, no display or network needed
+    ```
+
+    The accessibility detector needs system packages rather than an extra:
+
+    ```bash
+    apt install gir1.2-atspi-2.0 at-spi2-core gir1.2-gtk-3.0
+    pip install PyGObject
+    python -m tiny_computer_agent --doctor --detector best    # is the tree reachable?
     ```
 
     ## Run it on your machine
@@ -461,7 +519,61 @@ def readme():
     | `desktop.py` | Preflight checks, `doctor`, and `build_agent` in one call |
     | `demo.py` | The synthetic desktop used by tests and `--image example` |
     | `trace.py` | Portable traces: record a run with no pixels, replay it offline, score the choice against ground truth |
+    | `decoding.py` | Constrained decoding: mask the logits to a menu number or a yes/no, so an echo is impossible |
+    | `accessibility.py` | The accessibility tree: `AtspiReader` and `AtspiDetector`, roles and real extents |
     | `showcase.py` | The capability gallery: one real screenshot per capability, checked against the public API |
+    | `measure_vlm.py` | Measures the four ways to ask a model, on a real screen, and prints the table |
+
+    ## The marimo console
+
+    The loop is a library, but it is also a *session* you can drive one turn at a time -
+    which is what a console needs: a live screen preview, an action log, and
+    step / pause / resume.
+
+    <img src="gallery/console.png" width="900">
+
+    `AgentLoop.run()` is the batch form; `begin()`, `step_once()`, `pause()`, `resume()`
+    and `finished` are the stepwise one. `tests/test_stepwise.py` asserts the two cannot
+    drift apart - the same scenario has to produce the same episode either way. The
+    console cell in this notebook wires that to marimo widgets:
+
+    - **live preview** - the current screen with the element index drawn over it, so you
+      can see what the agent sees and what it thinks is clickable
+    - **action log** - one row per step: the action, and the outcome or the refusal
+    - **step / run / reset** - `mo.ui.run_button`, so an auto-refresh can never fire a
+      step by accident
+    - **pause** - respected *between* steps: the episode stays open, and a single step
+      can still be taken through it
+    - **panic** - creates the same kill-switch file the CLI checks, so the next step
+      refuses to run
+
+    Switch the screen to "this machine, live" and it is the real desktop, indexed through
+    OCR or through the accessibility tree.
+
+    ## What the machine already knows: the accessibility tree
+
+    OCR reads pixels and *guesses* which marks are widgets. An operating system's
+    accessibility tree simply knows: every application that publishes one advertises each
+    button with a role, a name, and its real screen extents.
+
+    <img src="gallery/atspi-index.png" width="860">
+
+    `AtspiDetector` reads that tree through PyGObject's `Atspi` typelib and keeps only the
+    roles worth clicking - a frame and a panel have names too, and offering them to a
+    model is how you get an agent that clicks the window furniture. The coordinates come
+    back as absolute screen pixels, so `absolute_coordinates` tells `IndexedScreen` not to
+    re-map them through a downscale.
+
+    <img src="gallery/desk-atspi.png" width="860">
+
+    That click was found **without reading a single pixel**: the window's own status label
+    reads `last action: Save` afterwards, and the app wrote `Save` to its marker file.
+    `--detector best` tries the tree first, then OCR, then the numbered grid.
+
+    **Honest limits.** AT-SPI is Linux-only here; **UI Automation is not implemented**, and
+    `accessibility_available("uia")` says exactly that rather than pretending - so an agent
+    on Windows keeps using OCR. Tk barely publishes a tree and a canvas publishes none,
+    which is why this is a detector and not a replacement.
 
     ## Capability gallery
 
@@ -1108,15 +1220,22 @@ def readme():
     - [x] A real click on a real window, with a verified outcome
     - [x] Blind-grid-click guard, echo detection, no-progress guard
     - [x] The harness suite running on real Windows (Server 2022 guest, no KVM — see above)
-    - [ ] Windows run of the *live desktop* test: the workflow is committed, but it needs a
-          GitHub mirror with Actions enabled — Hugging Face does not run GitHub Actions
-    - [ ] Make the OCR guard in `test_cli.py` skip, rather than fail, when no engine is installed
-    - [ ] Make the VLM reliable on a live screen (it echoes the prompt instead of choosing)
-    - [ ] Accessibility-tree detector (AT-SPI on Linux, UI Automation on Windows)
     - [x] Score episodes: replay a trace and count how often the choice was right - see [scoring a brain, offline](#scoring-a-brain-offline)
     - [x] A capability gallery: one real screenshot per capability, generated by `showcase.py`
-    - [ ] A **Windows** run of the gallery: the workflow builds it on `windows-latest`, but it needs a GitHub mirror with Actions enabled - Hugging Face does not run GitHub Actions
-    - [ ] marimo console: live screen preview, action log, pause / resume / step
+    - [x] The OCR guard skips instead of failing: one tested guard, `conftest.require_ocr`
+    - [x] Stop the 256M VLM echoing the prompt - masked decoding, measured in
+          [asking a tiny model in four ways](#asking-a-tiny-model-in-four-ways-measured)
+    - [x] The marimo console: live screen preview, action log, pause / resume / step
+    - [x] An accessibility-tree detector on Linux (AT-SPI), driving a real click with no OCR
+    - [ ] **Make the 256M VLM choose correctly.** Masking fixed the echo and the latency; the model
+          still picks the wrong item from a four-item menu of exact widgets. That is now a question
+          about weights, or about a better default (the rule-based planner), not about prompt shape.
+    - [ ] **UI Automation** (Windows) for the accessibility detector. The interface is here and
+          `accessibility_available("uia")` reports it as not implemented rather than pretending.
+    - [ ] Windows run of the *live desktop* test: the workflow is committed, but it needs a
+          GitHub mirror with Actions enabled — Hugging Face does not run GitHub Actions
+    - [ ] A **Windows** run of the gallery: the workflow builds it on `windows-latest` (job
+          `gallery-windows`), with the same GitHub-mirror caveat
 
     ## License
 
@@ -1258,7 +1377,8 @@ def publish_status(
                 "tiny_computer_agent_app.py": _notebook.read_text(encoding="utf-8"),
             }
             # the tool that builds the capability gallery, shipped with the card it fills
-            for _tool in ("showcase.py", "showcase_core.py", "showcase_catalog.py"):
+            for _tool in ("showcase.py", "showcase_core.py", "showcase_catalog.py",
+                          "measure_vlm.py"):
                 if (_root / _tool).is_file():
                     _files[_tool] = (_root / _tool).read_text(encoding="utf-8")
             _folders = [f for f in ("tiny_computer_agent", "tests", "gallery",
@@ -1455,5 +1575,157 @@ def scoreboard():
     return
 
 
+@app.cell
+def console_setup():
+    # The console's controls. This cell owns the mutable session, so the console below
+    # can re-run on every click without ever losing the episode in progress.
+    import tempfile
+
+    console_goal = mo.ui.text(value="click 'Save' to save the document", label="goal",
+                              full_width=True)
+    console_brain = mo.ui.dropdown(
+        options={"planner (rule-based, no weights)": "planner", "scripted clicks": "scripted"},
+        value="planner (rule-based, no weights)", label="brain")
+    console_screen = mo.ui.dropdown(
+        options={"replayed screenshot + real OCR": "image", "this machine, live": "live"},
+        value="replayed screenshot + real OCR", label="screen")
+    console_interval = mo.ui.refresh(options=[5, 10, 30], default_interval=10,
+                                     label="live preview")
+    console_step = mo.ui.run_button(label="▶ step")
+    console_run = mo.ui.run_button(label="⏩ run", kind="success")
+    console_reset = mo.ui.run_button(label="⟲ reset", kind="warn")
+    console_pause = mo.ui.switch(label="pause", value=False)
+    console_real_input = mo.ui.switch(label="real mouse (needs the allow env var)", value=False)
+    console_panic = mo.ui.switch(label="panic: create the kill switch", value=False)
+
+    console_session = {
+        "loop": None,
+        "signature": None,
+        "kill": str(Path(tempfile.gettempdir()) / "tiny-computer-agent.console.STOP"),
+    }
+
+    return (
+        console_brain,
+        console_goal,
+        console_interval,
+        console_panic,
+        console_pause,
+        console_real_input,
+        console_reset,
+        console_run,
+        console_screen,
+        console_session,
+        console_step,
+    )
+
+
+@app.cell
+def console(
+    console_brain,
+    console_goal,
+    console_interval,
+    console_panic,
+    console_pause,
+    console_real_input,
+    console_reset,
+    console_run,
+    console_screen,
+    console_session,
+    console_step,
+):
+    # The console: live screen preview, the action log, and step / pause / resume.
+    from PIL import ImageDraw
+
+    _ALLOW_ENV = "TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT"
+    _goal = console_goal.value.strip() or "click 'Save' to save the document"
+    _signature = (_goal, console_brain.value, console_screen.value,
+                  bool(console_real_input.value))
+    _session = console_session
+    _loop = _session["loop"]
+
+    # the panic switch drives the same kill-switch file the CLI checks
+    if console_panic.value:
+        Path(_session["kill"]).write_text("stop")
+    else:
+        Path(_session["kill"]).unlink(missing_ok=True)
+
+    if _loop is None or _session["signature"] != _signature:
+        if console_brain.value == "scripted clicks":
+            _brain = MockBrain([{"kind": "click", "label": "Save"},
+                                {"kind": "done", "reason": "clicked Save"}])
+        else:
+            _brain = PlannerBrain()
+        if console_screen.value == "this machine, live":
+            _screen = IndexedScreen(GrabScreen(monitor=1), OcrDetector())
+        else:
+            _screen = IndexedScreen(ImageScreen(fake_desktop()), OcrDetector())
+        _real = bool(console_real_input.value) and os.environ.get(_ALLOW_ENV) == "1"
+        _loop = AgentLoop(
+            screen=_screen, brain=_brain,
+            executor=PyAutoGuiExecutor() if _real else DryRunExecutor(),
+            policy=Policy(dry_run=not _real, step_budget=8, kill_switch_path=_session["kill"]),
+        )
+        _loop.begin(_goal)
+        _session.update(loop=_loop, signature=_signature)
+
+    # the controls: run_button is True only on the run its own click started, so an
+    # auto-refresh can never fire a step by accident
+    _loop.paused = bool(console_pause.value)
+    if console_reset.value:
+        _loop.begin(_goal)
+    if console_step.value:
+        _loop.paused = False          # a single step is allowed through a pause
+        _loop.step_once()
+    if console_run.value:
+        _loop.run(_goal, max_steps=8)  # respects the pause switch, and the goal
+
+    # a live peek at the screen, without taking a turn
+    _observation = _loop.observation
+    if console_screen.value == "this machine, live":
+        _observation = _loop.screen.capture()
+        _armed = console_interval.value          # referencing it arms auto-refresh
+
+    _preview = None
+    if _observation is not None and hasattr(_observation.frame, "convert"):
+        _preview = _observation.frame.convert("RGB").copy()
+        _pen = ImageDraw.Draw(_preview)
+        for _element in _observation.elements:
+            _pen.rectangle(
+                [_element.x, _element.y,
+                 _element.x + max(_element.width, 1), _element.y + max(_element.height, 1)],
+                outline=(255, 138, 0), width=2)
+            _pen.text((_element.x + 3, max(0, _element.y - 12)), _element.label[:30],
+                      fill=(210, 90, 0))
+
+    _rows = [{"step": _step.index,
+              "action": _step.action.describe() if _step.action else "",
+              "outcome": _step.error or (_step.result.summary() if _step.result else "")}
+             for _step in _loop.episode.steps]
+
+    _state = "finished" if _loop.finished else ("paused" if _loop.paused else "ready")
+    _mode = "REAL mouse" if not _loop.executor.dry_run else "dry run (nothing is clicked)"
+    console_status = mo.md(
+        f"**{_state}** · {len(_rows)} step(s) · {_mode} · budget {_loop.policy.step_budget} · "
+        f"stop: `{_loop.episode.stop_reason}`"
+    )
+    if console_real_input.value and os.environ.get(_ALLOW_ENV) != "1":
+        console_status = mo.vstack([
+            console_status,
+            mo.md(f"`{_ALLOW_ENV}` is not set, so this stays a dry run.").callout(kind="warn"),
+        ])
+
+    mo.vstack([
+        mo.hstack([console_goal, console_brain, console_screen, console_interval],
+                  justify="start", wrap=True),
+        mo.hstack([console_step, console_run, console_pause, console_reset,
+                   console_real_input, console_panic], justify="start", wrap=True),
+        console_status,
+        mo.image(_preview) if _preview is not None else mo.md("_no frame yet_"),
+        mo.ui.table(_rows, selection=None) if _rows else mo.md("_no actions yet_"),
+    ])
+
+    return
+
+
 if __name__ == "__main__":
     app.run()