Blame for tiny_computer_agent_app.py

# /// script
# requires-python = ">=3.13"
# dependencies = [
#     "pillow==13.0.0",
#     "rapidocr-onnxruntime==1.4.4",
# ]
# ///

import marimo

__generated_with = "0.25.2"
app = marimo.App(width="medium", auto_download=["html"])

with app.setup:
    import os
    from pathlib import Path

    import marimo as mo
    from huggingface_hub import HfApi

    # The project's public API, imported once so no cell has to duplicate it.
    from tiny_computer_agent import (
        AgentLoop,
        AuditLog,
        DryRunExecutor,
        Element,
        FakeScreen,
        GrabScreen,
        MockBrain,
        PlannerBrain,
        Policy,
        PyAutoGuiExecutor,
        Watchdog,
        gate_from_spec,
        verify_audit,
    )
    from tiny_computer_agent.actions import Action
    from tiny_computer_agent.brain import DecisionMenu
    from tiny_computer_agent.demo import fake_desktop
    from tiny_computer_agent.observe import ImageScreen
    from tiny_computer_agent.perception import IndexedScreen, OcrDetector
    from tiny_computer_agent.trace import Scenario, benchmark


@app.cell(hide_code=True)
def readme():
    readme_md = r'''

    ---
    title: Tiny Computer Agent
    emoji: 🖱️
    colorFrom: indigo
    colorTo: green
    license: mit
    tags:
      - computer-use
      - gui-agent
      - agentic-harness
      - local-first
      - tiny-models
      - offline
      - marimo
    ---

    # 🖱️ Tiny Computer Agent

    **An agentic harness that drives a real computer with a tiny, fully local model.**
    No cloud API, no API keys, no telemetry — and a core that imports nothing outside the
    Python standard library.

    > **Status 2.0.0: it drives a real desktop — on Linux and on Windows — and it learns.**
    > Screen capture, an OCR element index with real coordinates, a real mouse click, and a
    > *verified* outcome. A verified run can be written down as a **skill** and replayed with no
    > model at all; a queue of goals can be run, resumed after a kill, and reported on; an
    > **approval gate** can stand in front of every action; and what changed on screen is a
    > **measured** structural diff rather than a guess.
    > **All 74 capabilities have a real screenshot** in the [capability gallery](#capability-gallery) (Linux captures).
    > The loop can be driven from a [marimo console](#the-marimo-console) with a live preview, an
    > action log and pause / resume / step; elements can come from the
    > [accessibility tree](#what-the-machine-already-knows-the-accessibility-tree) instead of OCR;
    > and a run can be saved as a portable *trace*, replayed offline and **scored**, so
    > "which brain is right, and how often?" is a number in a table.
    > The suite passes on a live Linux desktop and inside a real Windows Server
    > 2022 guest; what is not solved is model reliability — see
    > [measured results](#measured-results).

    ## The idea, in one loop

    ```
        observe         think          act           verify
      +---------+    +---------+    +---------+    +---------+
      | screen  |--->|  tiny   |--->| mouse / |--->|  next   |
      |  frame  |    |  model  |    | keyboard|    |  frame  |
      +---------+    +---------+    +---------+    +----+----+
                                                        |
      <-------------------------------------------------+
            repeat until done, or the step budget runs out
    ```

    1. **Observe** — capture the screen, then build an **element index**: OCR (and,
       optionally, the OS accessibility tree) locates text and widgets and gives each one a
       label and a real screen coordinate.
    2. **Think** — the model is shown a *short numbered menu* built from that index and
       answers with one number (plus a verb). It never emits coordinates.
    3. **Act** — the harness maps the chosen label back to pixels and executes it.
    4. **Verify** — re-read the screen; stop if the same action repeats against an unchanged
       screen, and only call the episode a success if a verifier can see it worked.

    ## Measured results

    Everything below was run from this repo on CPU, dry-run unless stated. Nothing is
    aspirational.

    ### On a live desktop, with a real click

    A real Tk window (a Save dialog with Save/Cancel buttons) on a **real X server**
    (`Xvfb :99`), driven with real mouse input. The button sits at window `+120+120`,
    `x=20, y=155`, `110x40`, so its centre is `(195, 295)`:

    ```
    $ python -m tiny_computer_agent --live --real-input --verify-text Saved "click 'Save' to save the document"
    mode: REAL INPUT (mouse and keyboard live)  |  screen: real desktop (monitor 1)  |  brain: planner
    index: auto - OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline))

      step 0 | click label='Save' reason="matched 'Save' in the element index" | ok: clicked (195, 296)

    result: success - verified after 1 action(s)
    ```

    The click landed on the button, the app changed its own window to "Saved", and the
    verifier confirmed it before the episode was called a success. **Pixels in, a real click
    out, and evidence that it worked.**

    ### Re-confirmed end to end on a headless Linux box

    The whole path above was re-run from scratch on a headless Debian container (gVisor
    sandbox, 20 vCPU, no GPU, no physical display, no network at run time) behind a virtual
    X server. First the suite, headless:

    ```bash
    python -m pip install -e ".[desktop,ocr,dev]"
    pytest tests -q
    # 227 passed, 2 skipped in 18.7s          (that run was 0.3.0; 2.0 has grown this to 327)
    ```

    The two skips are the live-desktop tests. Give them a display and they run for real:

    ```bash
    Xvfb :99 -screen 0 1280x800x24 &
    export DISPLAY=:99
    TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py -q
    # tests/test_live_desktop.py::test_ocr_finds_a_real_button_at_the_right_pixels PASSED
    # tests/test_live_desktop.py::test_a_real_click_lands_on_the_button PASSED
    # 2 passed
    ```

    Then the same desktop, driven through the CLI with the mouse actually live:

    ```bash
    export TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1
    python -m tiny_computer_agent --live --real-input --verify-text Saved \
        "click 'Save' to save the document"
    ```

    ```
    goal: click 'Save' to save the document
    mode: REAL INPUT (mouse and keyboard live)  |  screen: real desktop (monitor 1)  |  brain: planner  |  kill switch: create /tmp/tiny-computer-agent.STOP to stop
    index: auto - OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline))

      step 0 | click label='Save' reason="matched 'Save' in the element index" | ok: clicked (194, 296)

    result: success - verified after 1 action(s)
    ```

    The window was drawn at `+120+120` with its Save button at `x=20, y=155, 110x40`, so the
    button centre is `(195, 295)`. The click at `(194, 296)` is *on the button*, not near it —
    and the app wrote `Save` back to its own marker file. Pixels in, a real click out, and the
    app's own side effect as the proof.

    `--doctor` on that box, with the virtual display attached:

    ```
    tiny-computer-agent preflight (Linux x86_64, python 3.13.11)

      [ok  ] display server: DISPLAY=:99
      [ok  ] pillow: installed
      [ok  ] screen capture: mss (fast, multi-monitor)
      [ok  ] input backend (pyautogui): installed
      [ok  ] element detector (auto): OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline))
      [ok  ] model runtime: not needed: the planner is rule-based (no weights, no download)
      [ok  ] execution mode: dry run (nothing is clicked)
      [ok  ] kill switch: create /tmp/tiny-computer-agent.STOP to abort the next action

    READY: the harness can drive this desktop.
    ```


    ### On a replayed screenshot

    ```
    $ python -m tiny_computer_agent --image example --brain planner
    elements detected: 5
      step 0 | click label='Save' ... -> pixel (295, 284)          # OCR found the button
    ```

    **SmolVLM-256M-Instruct** (256M parameters, CPU) read the button off the pixels and
    named the right target:

    ```
      step 0 | click label='Save' reason="the reply names 'Save'" -> pixel (295, 284)
    ```

    ### Scoring a brain, offline

    "Which brain is right, and how often?" needs no desktop, no model and no network,
    because a **trace** keeps every frame's element index and throws the pixels away. Record a
    run once; replay it against any brain, as often as you like.

    ```
    $ python -m tiny_computer_agent --image example --trace runs/save.json --expected Save
    ...
    trace: wrote runs/save.json (2 frames)
    score: correct   click 'Save' to save the document: expected 'Save', chose 'Save'
    ```

    A trace is JSON, so it diffs in review, commits next to the tests, and replays on a machine
    with nothing installed. Scoring a whole suite is a table:

    ```
    scoreboard: 7/9 correct (78%)
      planner (rule-based): 100%
      first-item (baseline): 33%
      oracle (always right): 100%
      correct   save the document: expected 'Save', chose 'Save'
      incorrect save the document: expected 'Save', chose 'Files' (chose 'Files', not 'Save')
      ...
    ```

    The number is the brain's **decision** - what it chose when it had to commit. Whether the
    harness *allowed* that decision to run is a separate axis: a click the policy refused is
    still the choice it was, and it is flagged `(harness refused it)` rather than passed off as
    a clean success. Keeping those two apart is the point.

    ### Asking a tiny model in four ways (measured)

    The card used to say the 256M VLM "echoes the prompt instead of choosing". That is
    reproducible - and so is the fix. A live GTK window, a menu of four exact widgets
    (Files, Terminal, Save, Cancel), and the goal `click 'Save' to save the document`:

    ```
    mode     echoed  seconds  outcome
    json     no          8.9  BrainError: the reply mentions 4 menu entries, so it looks like an echo
    choice   no          1.9  BrainError: the constrained reply was not a menu number: '1000'
    scan     no          1.4  click label='Files'   (the model said yes to the first candidate)
    rank     no          9.4  click label='Files'   (highest yes/no log-odds margin, +0.94)
    ```

    Four ways to ask, all of them the same weights:

    | mode | how the model is asked | answer space |
    |---|---|---|
    | `json` | reply with JSON, parsed leniently | the whole vocabulary |
    | `choice` | one masked generation for a menu number | a handful of digits |
    | `scan` | one masked yes/no question per item, first yes wins | two tokens |
    | `rank` | the same questions, ranked by the yes/no log-odds margin | two tokens |

    What the numbers actually say:

    - **The echo is fixed, structurally.** In `json` mode the model hands the prompt
      back (`"The goal is to click 'Save' to save the document..."`) and the harness
      refuses it. In the three masked modes the reply *cannot* be the prompt, because
      every one of its tokens is masked out of the logits. The failure stopped being
      *detected* and started being *not representable*.
    - **It is about six times faster** - 8.9 s of free text becomes 1.4 s for a
      one-token answer - because the model is no longer asked to write.
    - **Accuracy is not fixed, and the numbers say why.** The model answers now, and
      answers *wrongly*: it accepts `Files` for a goal that names `Save`, even when the
      answer space is two tokens and each candidate is judged on its own. A 256M model
      cannot do this task. That is a finding about the weights, not about the harness -
      which is why the rule-based planner is still the default brain.

    Reproduce it with `python measure_vlm.py --screen live --detector best --open-gtk`.

    ### Which brain drove which run

    The harness and the brain are separable, and it matters which one a result came from.
    None of the runs below used a cloud API; the two model brains run on CPU:

    | Run | Brain | Weights | Outcome |
    |---|---|---|---|
    | Live Linux desktop, real click | `PlannerBrain` (rule-based) | none | **success, verified** |
    | `/tmp/saveapp.py` window live | `PlannerBrain` | none | clicked `Save` at `(194, 296)` |
    | `--image example` | `PlannerBrain` | none | picked `Save`, pixel `(295, 284)` |
    | `--image example` | `VLMBrain` (SmolVLM-256M-Instruct) | 256M, CPU | picked `Save` |
    | Live Linux desktop | `VLMBrain` (SmolVLM-256M-Instruct) | 256M, CPU | echoed the prompt; **refused** by echo detection |
    | Live Linux desktop | `TextPlannerBrain` (Qwen2.5-0.5B) | 0.5B, CPU | clicked Cancel, then garbled JSON; budget exhausted |

    The rule-based planner is what carries the verified end-to-end runs. The tiny model is the
    open problem: it is reliable on a clean screenshot and unreliable on a busy live screen.

    ### What did *not* work, and what caught it

    - **The 256M VLM echoes the prompt on a busy live screen.** Asked to click Save on the
      live Tk window it replied with the prompt text back. The harness's echo detection
      refused it (`BrainError: the reply mentions 4 menu entries, so it looks like an echo of
      the prompt`) and stopped — **no blind click**. On the cleaner synthetic screenshot the
      same model chose correctly, so this is a prompt/robustness problem, not a wiring one.
    - **Qwen2.5-0.5B text planner over the OCR index is not reliable**: it clicked Cancel
      before Save, then recovered a garbled JSON reply, then found Save — budget exhausted.
    - **The grid fallback announced itself.** After the dialog closed, OCR found no text, the
      index degraded to 96 numbered cells, and the run printed
      `warning: no text was found on screen, so the index is 96 numbered grid cells and the
      agent is choosing blind` — instead of silently guessing.

    **327 tests collect** - 315 pass, 12 skip for want of a display or an AT-SPI typelib - including real OCR over a real image and (opt-in) live-desktop tests
    that click a real button.

    ## Verified on real Windows

    The Windows claims in this card are not taken on faith. A full Windows guest was built
    and the harness installed inside it, on a headless Linux host with **no KVM** — the same
    kind of sandbox the Linux runs above were produced in.

    > **Provenance.** The numbers below are quoted from that run, not re-measured here, and no
    > picture from it survives: the guest, its disk image and its screenshots went with the
    > container that had produced them. Everything above - the gallery, the live click, the
    > traces - is reproducible from this repository; the Windows results below are a record of
    > what that guest reported.

    **What was built.**

    ```
    host        headless Linux container (no /dev/kvm; software emulation only)
    hypervisor  QEMU 10.0.13  -accel tcg,thread=multi  -m 8G  -smp 8  -cpu Skylake-Client
    firmware    SeaBIOS (the OVMF/UEFI build would not auto-boot the installation media)
    guest       Windows Server 2022 Standard Evaluation (Desktop Experience), x86_64
    install     unattended from the public evaluation ISO, install.wim index 2,
                answer file on removable media, autologon as Administrator
    ```

    The image-apply phase moved roughly 9 GB to disk in about thirteen minutes, and the
    desktop was up a few minutes after that.

    **What was installed inside it.**

    ```
    Python 3.12.8 (embeddable distribution, unpacked to C:\tiny\py)
    pytest, pillow 13.0.0, mss 10.2.0
    numpy 2.5.4, onnxruntime 1.31.1, opencv-python 5.0.0.93, rapidocr-onnxruntime 1.4.4
    the harness itself, installed into site-packages
    ```

    **Results.**

    | Configuration | Result | Wall clock |
    |---|---|---|
    | Harness suite, no OCR engine | **118 passed, 3 failed, 4 skipped** | 7.8 s |
    | Harness suite, OCR engine installed, `test_perception.py` excluded | **109 passed, 3 skipped, exit 0** | 21 min |
    | `test_perception.py` (OCR inference) | runs; roughly ten minutes per OCR pass | — |

    None of the three failures in the first row is a harness defect: two are
    `ModuleNotFoundError: numpy` and the third is the guard bug documented below. With the
    OCR engine installed, the same suite exits **0**. (Those Windows counts predate `trace.py` and 2.0; the harness suite has since grown to 327 tests.)

    **The emulated-OCR caveat.** onnxruntime requires AVX2, and QEMU's software emulation of
    AVX2 is roughly two orders of magnitude slower than native. A suite that finishes in 17
    seconds on Linux takes over half an hour on this guest, and the time is spent entirely
    inside `onnxruntime_inference_collection.py`. That is a property of emulation, not of the
    harness: on real hardware — including a GitHub `windows-latest` runner — it does not
    apply.

    ### What running on Windows caught

    Four things surfaced by actually running there. Three are genuine deployment requirements
    that no Linux-only test run can reveal:

    | Finding | Symptom | Fix |
    |---|---|---|
    | **onnxruntime needs AVX2** | `Windows fatal exception: code 0xc000001d` (`STATUS_ILLEGAL_INSTRUCTION`) the moment `onnxruntime` is imported, because QEMU's default `qemu64` CPU advertises no AVX | boot the guest with `-cpu Skylake-Client` (any AVX2 model will do) |
    | **onnxruntime needs the MSVC runtime** | `ImportError: DLL load failed while importing onnxruntime_pybind11_state` on a clean Server 2022 | install `vc_redist.x64.exe` in the guest |
    | **`pyautogui` ships as an sdist only** | `pip install pytest pillow mss pyautogui` fails as a whole, so `pytest` never arrives and the suite reports `No module named pytest` | install `pyautogui` in a separate step from the wheel-only packages |
    | **A guard that never fires** | `test_screenshot_replay_needs_no_display` fails instead of skipping when no OCR engine is present: it guards on `detector_available()[0]`, which is `True` for the grid fallback | guard on the OCR detector specifically |

    The last entry is a defect in this repository rather than in the environment, and it is
    exactly the kind of thing that only surfaces on a machine without an OCR engine.

    ## The test suite, in full

    `pytest tests` collects **327 tests**. 325 run headless — no display, no network, no
    weights. The last 2 are the opt-in live-desktop tests, skipped unless
    `TINY_COMPUTER_AGENT_LIVE_TEST=1`. On a box without PyGObject, two of the 282 *fail* rather
    than skip: they assert the AT-SPI reader is reachable, and that is the honest outcome.

    | File | Tests | What it proves |
    |---|---:|---|
    | `tests/test_actions.py` | 13 | the closed vocabulary — every action is validated before anything executes |
    | `tests/test_act.py` | 8 | `DryRunExecutor` is inert; the real executor refuses until explicitly armed |
    | `tests/test_observe.py` | 8 | element/observation geometry, `FakeScreen`, `ImageScreen`, the capture fallback |
    | `tests/test_perception.py` | 13 | OCR labels a real image with real coordinates; the grid fallback; `AutoDetector` |
    | `tests/test_brain.py` | 18 | the numbered `DecisionMenu`, lenient parsing, echo detection, `MockBrain`, `PlannerBrain` |
    | `tests/test_loop.py` | 14 | `AgentLoop` / `Episode` / `Step`, the no-progress guard, the step budget |
    | `tests/test_verify.py` | 7 | success is only claimed when a verifier can actually see it |
    | `tests/test_safety.py` | 11 | deny lists, the kill switch, the blind-grid-click guard |
    | `tests/test_grid_fallback.py` | 11 | the fallback index announces itself instead of guessing |
    | `tests/test_desktop.py` | 12 | display detection, the preflight checks, `doctor` |
    | `tests/test_cli.py` | 22 | the command line: `--doctor`, `--image`, `--live`, and the 2.0 `skills` / `batch` subcommands |
    | `tests/test_trace.py` | 28 | traces: a run recorded with no pixels, replayed against another brain, and scored against ground truth |
    | `tests/test_models.py` | 7 | the model backends: lazy loading, device choice, honest refusals |
    | `tests/test_api_surface.py` | 4 | the public API is a promise: everything advertised is importable, and nothing public is left out |
    | `tests/test_decoding.py` | 24 | the mask, the answer space, the retry ladder, the vote - and that an echo is refused |
    | `tests/test_accessibility.py` | 27 | role filtering, the coordinate contract, the fallback chain, and real AT-SPI |
    | `tests/test_diff.py` | 8 | what changed between two frames: added, removed, moved, and the verifier that remembers |
    | `tests/test_stepwise.py` | 11 | begin / step / pause / resume, and that it equals the batch loop |
    | `tests/test_ocr_guard.py` | 4 | the OCR guard really skips with no engine - the guard that never fired |
    | `tests/test_gates.py` | 14 | the operator's yes: whitelists, ask-once gates, and a refusal that is a step |
    | `tests/test_skills.py` | 17 | only a verified run becomes a skill; re-grounding by label; aiming at nothing raises |
    | `tests/test_runner.py` | 10 | a queue, a JSONL resume file, and a report counts what was verified |
    | `tests/test_watchdog.py` | 12 | a hung call becomes a named step instead of a stall - and the thread it leaves behind is admitted |
    | `tests/test_audit.py` | 12 | the chain verifies, and editing, deleting or reordering a line is caught by number |
    | `tests/test_atspi_guard.py` | 4 | the AT-SPI guard really skips without the typelib - the same trap as the OCR guard |
    | `tests/test_live_desktop.py` | 2 | **needs a display, opt-in**: OCR finds a real button at the pixels it was drawn at, and a real click makes the app react |

    ```bash
    pytest tests -q                                  # 274 passed, 8 skipped, 2 failed only where
                                                     # PyGObject/AT-SPI is missing (gir1.2-atspi-2.0)
    TINY_COMPUTER_AGENT_LIVE_TEST=1 pytest tests -q  # + the 2 live-desktop tests, on a desktop
    ```

    ## Install

    ```bash
    pip install -e ".[ocr]"              # labels from pixels (RapidOCR: no system binary)
    pip install -e ".[desktop,ocr]"      # + real capture and real input
    pip install -e ".[vision,ocr]"       # + the tiny model brains
    pytest                               # 327 tests, no display or network needed
    ```

    The accessibility detector needs system packages rather than an extra:

    ```bash
    apt install gir1.2-atspi-2.0 at-spi2-core gir1.2-gtk-3.0
    pip install PyGObject
    python -m tiny_computer_agent --doctor --detector best    # is the tree reachable?
    ```

    ## Run it on your machine

    ```bash
    python -m tiny_computer_agent --doctor                 # what can this machine do?
    python -m tiny_computer_agent --image example          # replay a screenshot, dry run
    python -m tiny_computer_agent --live                   # your real screen, dry run
    python -m tiny_computer_agent --live --real-input "click 'OK'"
    python -m tiny_computer_agent --live --real-input --verify-text "Saved" "click 'Save'"
    ```

    `--doctor` is the first thing to run on a new machine: it checks the display session,
    capture backend, pyautogui, the OCR engine, the model runtime and the kill switch, and
    prints the fix for anything missing. Real input needs **two** deliberate acts:

    ```bash
    export TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1
    ```

    **Pass `--verify-text`.** Without it, a successful action that removes its own target
    (clicking Save closes the dialog) is indistinguishable from a failure, and the episode is
    recorded as one. With it, the harness only reports success when it can see the text it
    was given.

    ### No desktop? Use a virtual one

    Xvfb is enough for everything except a real model's hallucinations:

    ```bash
    Xvfb :99 -screen 0 1280x800x24 &
    export DISPLAY=:99
    TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py
    ```

    That test starts its own Tk window, asserts OCR finds the button at the coordinates it
    was actually drawn at, clicks it for real, and checks the app reacted. It is opt-in
    because it moves your mouse — run it on a throwaway desktop.

    ### Testing on Windows - three ways, and which to use

    The verified Windows run above used method 2. All three remain valid:

    | Method | Real Windows desktop? | Cost | Status |
    |---|---|---|---|
    | **GitHub Actions `windows-latest`** | yes, interactive session | free for public repos, minutes | workflow committed at `.github/workflows/live-desktop.yml` |
    | **QEMU/TCG VM on a Linux host** | yes, fully emulated | ~25 min to install, hours to run OCR | **used for the results above** |
    | **A cloud Windows box over RDP** | yes, native | paid by the hour | untested here |

    Nothing in the harness is Linux-specific: `pyautogui` uses `SendInput` on Windows and
    `GrabScreen` already falls back to Pillow's `ImageGrab` there. Two things can only fail on
    a real Windows desktop, which is why these runs exist: **DPI scaling** (that is what
    `--scale` is for) and the native input backend. At 125% or 150% display scaling, expect the
    coordinate assertion in `test_live_desktop.py` to be the first thing to fail - that is the
    signal, not a nuisance.

    **Reproducing the local VM.** With no KVM, a Windows guest is still perfectly possible -
    it is simply slow. The working recipe, including the two ingredients that onnxruntime
    demands:

    ```bash
    # 1. install media, plus an answer file on removable media
    qemu-img create -f qcow2 win.qcow2 64G
    qemu-system-x86_64 -accel tcg,thread=multi -m 8G -smp 8 -machine pc \
      -cpu Skylake-Client \
      -drive file=win.qcow2,if=none,id=hd0,format=qcow2,cache=writeback \
      -device ide-hd,drive=hd0,bus=ide.0 \
      -drive file=win2022.iso,if=none,id=cd0,media=cdrom,readonly=on \
      -device ide-cd,drive=cd0,bus=ide.1 \
      -drive if=none,id=usb,format=raw,file=unattend.img \
      -device usb-ehci,id=ehci -device usb-storage,bus=ehci.0,drive=usb \
      -netdev user,id=n0 -device e1000,netdev=n0 \
      -vga std -display none -vnc :1 -boot order=d,menu=off

    # 2. inside the guest, after autologon
    #    vc_redist.x64.exe is required before onnxruntime will load
    curl.exe -s -o C:\tiny\vc_redist.x64.exe <vc_redist-url> && C:\tiny\vc_redist.x64.exe /install /quiet /norestart
    ```

    Serve files to the guest over QEMU's user-mode networking (the host is `10.0.2.2`), which
    avoids typing long commands into the emulated console. Two practical notes from doing
    this: an answer file with non-canonical XML namespaces is **silently ignored** by Setup
    (you land on the language page), and keyboard input over VNC **drops the shift modifier**
    (`_` becomes `-`, `:` becomes `;`) - the QEMU monitor's `sendkey` handles shifts correctly.


    ## How it works

    | Module | Job |
    |---|---|
    | `actions.py` | The closed vocabulary — `click` `type` `key` `scroll` `wait` `done`, validated before anything executes |
    | `observe.py` | `Element`/`Observation`, `FakeScreen`, `ImageScreen`, `GrabScreen` (mss → Pillow fallback) |
    | `perception.py` | Detectors: `OcrDetector` (RapidOCR), `GridDetector`, `AutoDetector`, `IndexedScreen` |
    | `brain.py` | `DecisionMenu` (the numbered menu), `ModelBrain`, lenient `parse_decision`, echo detection, `MockBrain`, `PlannerBrain` |
    | `models.py` | `VLMBrain` (SmolVLM-256M) and `TextPlannerBrain` (Qwen2.5-0.5B), lazily loaded |
    | `act.py` | `DryRunExecutor` (the default) and `PyAutoGuiExecutor` |
    | `safety.py` | `Policy`: deny lists, kill switch, step budget, the blind-grid-click guard |
    | `loop.py` | `AgentLoop`, `Episode`, `Step`, the no-progress guard and the verifier |
    | `desktop.py` | Preflight checks, `doctor`, and `build_agent` in one call |
    | `demo.py` | The synthetic desktop used by tests and `--image example` |
    | `trace.py` | Portable traces: record a run with no pixels, replay it offline, score the choice against ground truth |
    | `decoding.py` | Constrained decoding: mask the logits to a menu number or a yes/no, so an echo is impossible |
    | `accessibility.py` | The accessibility tree: `AtspiReader` and `AtspiDetector`, roles and real extents |
    | `diff.py` | What changed between two frames: `Delta` (added / removed / moved) and `ChangeVerifier`, so "the dialog closed" is checkable |
    | `skills.py` | **2.0** - Skills: a *verified* run written down once, replayed by `SkillBrain` with no model. Labels, never coordinates |
    | `gates.py` | **2.0** - Approval gates: `AllowOnlyLabels`, `InteractiveGate` (yes / no / always / never), `AllOf` / `AnyOf`. A refusal is a step |
    | `runner.py` | **2.0** - `TaskRunner`: a queue of goals, a JSONL resume file, and a `RunReport` that counts what was *verified* |
    | `watchdog.py` | **2.0** - `Watchdog`: a deadline on `decide` / `execute` / `capture`, so a hung call is a step with a reason, not a stall |
    | `audit.py` | **2.0** - `AuditLog`: an append-only, hash-chained record of a run. Edit, delete or reorder a line and `verify_audit` names it |
    | `showcase.py` | The capability gallery: one real screenshot per capability, checked against the public API |
    | `measure_vlm.py` | Measures the four ways to ask a model, on a real screen, and prints the table |
    | `evaluate_daily.py` | A day's tasks against a real app, verified by the app's own log |

    ## The marimo console

    The loop is a library, but it is also a *session* you can drive one turn at a time -
    which is what a console needs: a live screen preview, an action log, and
    step / pause / resume.

    <img src="gallery/console.png" width="900">

    `AgentLoop.run()` is the batch form; `begin()`, `step_once()`, `pause()`, `resume()`
    and `finished` are the stepwise one. `tests/test_stepwise.py` asserts the two cannot
    drift apart - the same scenario has to produce the same episode either way. The
    console cell in this notebook wires that to marimo widgets:

    - **live preview** - the current screen with the element index drawn over it, so you
      can see what the agent sees and what it thinks is clickable
    - **action log** - one row per step: the action, and the outcome or the refusal
    - **step / run / reset** - `mo.ui.run_button`, so an auto-refresh can never fire a
      step by accident
    - **pause** - respected *between* steps: the episode stays open, and a single step
      can still be taken through it
    - **panic** - creates the same kill-switch file the CLI checks, so the next step
      refuses to run

    Switch the screen to "this machine, live" and it is the real desktop, indexed through
    OCR or through the accessibility tree.

    ## What the machine already knows: the accessibility tree

    OCR reads pixels and *guesses* which marks are widgets. An operating system's
    accessibility tree simply knows: every application that publishes one advertises each
    button with a role, a name, and its real screen extents.

    <img src="gallery/atspi-index.png" width="860">

    `AtspiDetector` reads that tree through PyGObject's `Atspi` typelib and keeps only the
    roles worth clicking - a frame and a panel have names too, and offering them to a
    model is how you get an agent that clicks the window furniture. The coordinates come
    back as absolute screen pixels, so `absolute_coordinates` tells `IndexedScreen` not to
    re-map them through a downscale.

    <img src="gallery/desk-atspi.png" width="860">

    That click was found **without reading a single pixel**: the window's own status label
    reads `last action: Save` afterwards, and the app wrote `Save` to its marker file.
    `--detector best` tries the tree first, then OCR, then the numbered grid.

    **Honest limits.** AT-SPI is Linux-only here. UI Automation *is* implemented - `UiaReader` and
    `UiaDetector`, unit-tested against a synthetic control tree - but it has never run against a
    real Windows desktop, and `accessibility_available("uia")` reports the truth on any machine
    rather than pretending, so an agent on Windows still falls back to OCR. Tk barely publishes a tree and a canvas publishes none,
    which is why this is a detector and not a replacement.

    ## Can it do a day's work?

    The honest way to answer "could this replace a person at a PC?" is to give it a
    person's tasks and check the result against **the application's own record**, not the
    agent's opinion of itself. Eleven of them, on a real GTK desktop with the real mouse:

    ```
    button          yes   0.7  Click Save
    right-button    yes   0.4  Click Cancel, not the button next to it
    type            yes   1.8  Type a new file name and save
    shortcut        yes   0.7  Press Ctrl+S
    menu            yes   0.7  Open the File menu and choose Save As
    scroll          yes   1.5  Scroll the list and pick a row
    second-window   yes   0.7  Open Settings and press Apply there
    see             yes   0.1  Read what the window says
    verify          yes   0.7  Check the save actually happened
    refuse          yes   0.1  Do not press the Terminal button
    kill-switch     yes   0.1  Stop a live run mid-plan

    11/11 tasks completed, verified by the application's own log
    ```

    Run it yourself with `python evaluate_daily.py` (`--only menu` for one task).

    **What that number does and does not mean.** The first run of this suite scored
    **5/11**, and the failures were not the model. A coordinate bug in the detector
    *chain* was shifting every accessibility click by the capture's scale factor:
    `--detector best` wraps the tree detector inside `AutoDetector`, which reported
    "frame coordinates" for elements that were already screen pixels, so `IndexedScreen`
    re-mapped them through the downscale and the click landed beside the button. Two more
    real gaps surfaced with it - an unnamed GTK list row was invisible to the index (so
    nothing in a list could be clicked at all), and two GTK instances sharing an
    application id made the second launch silently do nothing. All three are fixed and
    pinned by tests.

    So the machine half works: it can see, decide, act and verify a day's operations, and
    it refuses the dangerous ones. What this suite deliberately does **not** measure is
    *choosing* the actions - nine of the eleven tasks were driven by a scripted action
    list, because that is what isolates the harness from the brain. Choosing is measured
    in [asking a tiny model in four ways](#asking-a-tiny-model-in-four-ways-measured):
    4/4 goals right for a 0.5B text planner over the accessibility index in `rank` mode,
    and 1-2/4 for a 256M VLM.

    Caveats, stated plainly: this is a Linux GTK desktop under Xvfb with **no window
    manager** (so the keyboard tasks had to move the pointer into the window first,
    because X without a manager focuses by pointer); Windows was not exercised; the tasks
    are single-application with no overlapping windows, no animations and no network; and
    eleven tasks are a sample, not a survey.

    ## Capability gallery

    **74 capabilities, one screenshot each - and 163 of the package's 270 public callables (functions, methods, classes and protocols) appear in at least one of them**, together with 8 of its 26 public constants. **Not yet pictured: 107 callables** - 103 in the 2.0 operator layer (skills 29, the batch runner 20, approval gates 19, the structural diff 14, the audit log 13, the watchdog 8) and four CLI entry points. They are covered by tests rather than by pictures, and `showcase.py` prints the exact list. The counts are read off `gallery/index.json`, which records what each picture shows, and checked against the package by `showcase.api_surface()`.

    Nothing here is a mock-up. Each picture is produced by [`showcase.py`](showcase.py): a command or a call is run for real, and its actual output - or the real desktop it acted on - is what you see. The desktop shots are real screen captures on a real X server (Xvfb, 1280x800), including a real mouse click verified by the target application's own reaction.

    > **An honest note on the platform.** These were captured on Linux, because the sandbox that builds this card has no Windows: no KVM, no QEMU and no Windows image. The harness is OS-agnostic (pyautogui uses `SendInput` on Windows, and `GrabScreen` falls back to Pillow's `ImageGrab`), so these are the same code paths - but they are not Windows pixels. There are no Windows pictures here, and that workflow does not by itself produce any: its `windows-latest` job does build the same gallery on real Windows pixels, but it uploads them as a CI **artifact** (`gallery-windows`) - nothing downloads it, re-prefixes the files `win-*`, or appends a section to this card, so an artifact is as far as they get. The Windows screenshots from the guest described [below](#verified-on-real-windows) were lost with the build container that produced them.

    Regenerate it with `python showcase.py` (`--list` to see the ids, `--only <id>` for a subset, `--card` to rewrite this section). It runs the calls for real, so a broken capability fails loudly instead of shipping a stale picture.

    | area | capabilities |
    |---|---:|
    | Action vocabulary | 3 |
    | Decisions | 7 |
    | Execution | 2 |
    | The loop | 7 |
    | Safety | 4 |
    | Screens | 6 |
    | Perception | 6 |
    | Preflight | 4 |
    | Models | 5 |
    | Traces & scoring | 7 |
    | Command line | 9 |
    | Real desktop | 7 |
    | Extensibility | 1 |

    ---

    ### Action vocabulary

    **The six verbs, and only six**

    *Every action is validated on construction, before anything touches the machine.*

    <img src="gallery/vocab-verbs.png" width="860">

    `shows:` `ACTION_KINDS`, `ACTION_MENU`, `Action`, `Action.describe`, `Action.to_dict`, `parse_action`

    **Malformed actions are refused, not guessed at**

    *An unknown verb or a missing target raises before the machine is touched.*

    <img src="gallery/vocab-refusals.png" width="860">

    `shows:` `ActionError`, `Action`, `parse_action`

    **The contract handed to a model**

    *The prompt-side menu: one JSON object, six shapes, no invented verbs.*

    <img src="gallery/vocab-menu.png" width="860">

    `shows:` `ACTION_MENU`

    ### Decisions

    **A short numbered menu, in reading order**

    *A 256M model cannot emit coordinates; it can pick a number from twelve.*

    <img src="gallery/menu-numbered.png" width="860">

    `shows:` `DecisionMenu`, `DecisionMenu.render`, `MenuItem`, `MenuItem.label`

    **Menu lookup tolerates what models actually send**

    *1, "2", 2.0 and "Save." all resolve; a stray word resolves to nothing.*

    <img src="gallery/menu-lookup.png" width="860">

    `shows:` `DecisionMenu.item`, `DecisionMenu.find`, `DecisionMenu.mentioned_in`

    **Reading a decision out of a small model's reply**

    *The LAST JSON object wins, because a model that echoes its prompt answers last.*

    <img src="gallery/decision-json.png" width="860">

    `shows:` `parse_decision`, `extract_json_object`, `extract_json_objects`, `DecisionMenu`

    **An echoed prompt is refused; a right answer in the wrong field is recovered**

    *Real 256M/0.5B failure modes, both caught: no blind click either way.*

    <img src="gallery/decision-echo-guard.png" width="860">

    `shows:` `parse_decision`, `BrainError`, `DecisionMenu.mentioned_in`

    **A garbled verb with a valid label is still usable**

    *A 256M model really does emit {"kind": "file", "label": "Save"}.*

    <img src="gallery/decision-garbled.png" width="860">

    `shows:` `parse_decision`, `BrainError`

    **MockBrain: replay a fixed script, exactly once**

    *Deterministic and offline - the workhorse for tests, demos and replays.*

    <img src="gallery/brain-mock.png" width="860">

    `shows:` `MockBrain`, `MockBrain.decide`, `MockBrain.remaining`

    **PlannerBrain: no weights, no download, and it gives up honestly**

    *The proof the harness works end to end, and a real fallback when no model is wanted.*

    <img src="gallery/brain-planner.png" width="860">

    `shows:` `PlannerBrain`, `PlannerBrain.decide`, `GoalUnreachable`

    ### Execution

    **Dry run resolves every click to pixels - and touches nothing**

    *The default everywhere: read a whole episode before anything clicks.*

    <img src="gallery/exec-dry-run.png" width="860">

    `shows:` `DryRunExecutor`, `DryRunExecutor.execute`, `resolve_point`, `ActionResult`

    **Real input refuses to guess a pixel for a label it cannot find**

    *This is the gate between a hallucinated label and 'Delete account'.*

    <img src="gallery/exec-refuses-unresolved.png" width="860">

    `shows:` `ActionResult.summary`, `InputUnavailable`, `PyAutoGuiExecutor`, `PyAutoGuiExecutor.execute`

    ### The loop

    **observe -> think -> act -> verify, traced end to end**

    *One call runs a whole episode; every step is kept for audit.*

    <img src="gallery/loop-end-to-end.png" width="860">

    `shows:` `AgentLoop`, `AgentLoop.run`, `Episode`, `Episode.render`, `Episode.to_rows`, `Step`, `Step.line`, `Step.ok`

    **A hard step budget, so a run cannot wander**

    *The budget ends the episode and says so; nothing executes past it.*

    <img src="gallery/loop-budget.png" width="860">

    `shows:` `Policy`, `Policy.step_budget`, `AgentLoop.run`

    **Repeating one action against an unchanged screen is stopped**

    *A stuck agent is not a persistent one - it is stopped and told why.*

    <img src="gallery/loop-no-progress-guard.png" width="860">

    `shows:` `AgentLoop.run`, `Step`, `Step.error`

    **A kill-switch file aborts before the next action**

    *Checked before every single action: create the file, the run stops.*

    <img src="gallery/loop-kill-switch.png" width="860">

    `shows:` `Policy.kill_switch_path`, `Policy.should_stop`, `AgentLoop.run`

    **A dead screen or a broken brain is an episode, not a traceback**

    *Every failure path is recorded as a failed step with a reason.*

    <img src="gallery/loop-survives-failures.png" width="860">

    `shows:` `AgentLoop.run`, `Episode.stop_reason`, `GoalUnreachable`, `BrainError`

    **Success is evidence, not the model's word**

    *Without a verifier, a successful action that removes its own target looks like a failure.*

    <img src="gallery/loop-verifier.png" width="860">

    `shows:` `AgentLoop.run`, `Episode.success`, `Observation.text`

    **Every step can be streamed out as it happens**

    *This is the hook a live console, a progress bar or a websocket uses.*

    <img src="gallery/loop-on-step.png" width="860">

    `shows:` `AgentLoop.run`, `Step.line`, `on_step`

    ### Safety

    **Typing from the deny list never reaches the keyboard**

    *Checked before execution; a refusal ends the episode and clicks nothing.*

    <img src="gallery/safety-deny-text.png" width="860">

    `shows:` `Policy`, `Policy.check`, `PolicyViolation`, `DEFAULT_DENY_TEXT`

    **Clicking a denied widget is refused, however the label is spelled**

    *Terminal, keychain, wallet, password manager, bank - matched on the label.*

    <img src="gallery/safety-deny-label.png" width="860">

    `shows:` `Policy.deny_labels`, `Policy.check`, `Observation.find`

    **Key chords on the deny list are never pressed**

    *ctrl+alt+t, ctrl+alt+delete and ctrl+shift+escape are refused by default.*

    <img src="gallery/safety-deny-keys.png" width="860">

    `shows:` `Policy.deny_keys`, `Policy.check`, `PolicyViolation`

    **A numbered grid cell is a blind click: allowed to plan, refused for real**

    *With real input the blind guess is refused unless you pass --allow-grid.*

    <img src="gallery/safety-blind-grid.png" width="860">

    `shows:` `Policy.allow_grid_clicks`, `Policy.check`, `GridDetector`

    ### Screens

    **A scripted screen, for testing the loop with no display**

    *The final frame repeats forever, so a runaway agent cannot exhaust the script.*

    <img src="gallery/screen-fake.png" width="860">

    `shows:` `FakeScreen`, `FakeScreen.capture`, `Observation`

    **A still image (or a sequence) replayed as a screen**

    *Lets the whole pipeline be exercised with no display, and a failure be replayed.*

    <img src="gallery/screen-image.png" width="860">

    `shows:` `ImageScreen`, `ImageScreen.capture`, `fake_desktop`

    **Real capture: monitors, downscaling, and HiDPI mapping**

    *Frame pixel -> screen pixel is preserved, so a click cannot drift.*

    <img src="gallery/capture-options.png" width="860">

    `shows:` `GrabScreen`, `GrabScreen.capture`, `GrabScreen.map_to_screen`, `ScreenUnavailable`

    **Labels are compared on their alphanumeric core**

    *OCR loses spaces, models add punctuation - so nothing matches by exact string.*

    <img src="gallery/element-matching.png" width="860">

    `shows:` `Element`, `Element.center`, `Element.matches`, `Observation.find`, `normalise_label`

    **The compact text a model is shown**

    *This is the whole world as the brain sees it - capped, and honest when empty.*

    <img src="gallery/observation-summary.png" width="860">

    `shows:` `Observation.summary`

    **The built-in synthetic desktop**

    *Draws a screenshot-like desktop offline - the same image --image example replays.*

    <img src="gallery/fake-desktop.png" width="860">

    `shows:` `fake_desktop`, `write_fake_desktop`, `OcrDetector`

    ### Perception

    **Probing for a detector, and building one by name**

    *The doctor and the CLI both ask this before promising anything.*

    <img src="gallery/detector-available.png" width="860">

    `shows:` `detector_available`, `make_detector`, `DetectorUnavailable`, `Detector`

    **OCR turns a screenshot into labelled, clickable elements**

    *The model never has to guess where anything is: this is the grounding.*

    <img src="gallery/ocr-labels.png" width="860">

    `shows:` `OcrDetector`, `OcrDetector.detect`, `Element`

    **Three knobs keep dialog prose out of the menu**

    *Every extra line of noise is one more chance for a small model to click prose.*

    <img src="gallery/ocr-filters.png" width="860">

    `shows:` `OcrDetector.min_score`, `OcrDetector.max_label_chars`, `OcrDetector.min_label_chars`

    **A numbered grid, for screens with no text at all**

    *An agent with an empty index has no legal moves; a grid always has moves.*

    <img src="gallery/grid-detector.png" width="860">

    `shows:` `GridDetector`, `GridDetector.detect`

    **OCR when there is text, a grid when there is none**

    *The default: useful with nothing installed, better the moment OCR appears.*

    <img src="gallery/auto-detector.png" width="860">

    `shows:` `AutoDetector`, `AutoDetector.detect`

    **IndexedScreen: capture + detect, composed**

    *One capture() call gives the loop an index it can act on.*

    <img src="gallery/indexed-screen.png" width="860">

    `shows:` `IndexedScreen`, `IndexedScreen.capture`, `Observation.summary`

    ### Preflight

    **Is there a desktop session to drive at all?**

    *The first question the CLI and the doctor both ask.*

    <img src="gallery/display-available.png" width="860">

    `shows:` `display_available`

    **Every precondition, one list, each with the fix**

    *Eight checks; a required one that fails is a blocking problem.*

    <img src="gallery/run-checks.png" width="860">

    `shows:` `run_checks`, `Check`, `Check.render`

    **python -m tiny_computer_agent --doctor**

    *The first thing to run on a new machine. Exits non-zero when not ready.*

    <img src="gallery/cli-doctor.png" width="860">

    `shows:` `detector_available`, `display_available`, `doctor`, `report`

    **One call wires screen + detector + brain + executor + policy**

    *Defaults are the safe ones: dry run, a step budget, and a kill-switch file.*

    <img src="gallery/build-agent.png" width="860">

    `shows:` `build_agent`, `Policy`, `AgentLoop`

    ### Models

    **Which model runtime is present, and are the weights cached?**

    *Weights are loaded lazily, so asking is free.*

    <img src="gallery/models-deps.png" width="860">

    `shows:` `model_dependencies`, `model_cached`, `MODEL_BRAINS`, `make_model_brain`, `DEFAULT_TEXT_MODEL`, `DEFAULT_VISION_MODEL`, `BrainError`

    **Device and dtype selection**

    *CUDA if it is there, CPU otherwise; the harness never needs a GPU.*

    <img src="gallery/models-device.png" width="860">

    `shows:` `resolve_device`, `pick_dtype`

    **The prompt a model actually receives - and how it learns from a dead end**

    *Placeholders, not worked examples: a weak model echoes the prompt, and an echo fails validation.*

    <img src="gallery/models-prompt.png" width="860">

    `shows:` `ModelBrain`, `ModelBrain.build_prompt`, `ModelBrain.decide`

    **A second decision is told what already failed**

    *'you already tried this and the screen did NOT change' is the most useful line a small model gets.*

    <img src="gallery/models-prompt-history.png" width="860">

    `shows:` `ModelBrain`, `ModelBrain.build_prompt`, `ModelBrain.history`

    **The two real model backends, and lazy loading**

    *Constructing a brain costs nothing; the weights arrive on the first decide().*

    <img src="gallery/models-swap.png" width="860">

    `shows:` `TextPlannerBrain`, `VLMBrain`, `TextPlannerBrain.unload`, `VLMBrain.unload`, `ModelBrain.loaded`, `make_model_brain`

    ### Traces & scoring

    **A task with one right answer, and the frames to show**

    *Ground truth lives with the task, so a run can be scored at all.*

    <img src="gallery/scenario.png" width="860">

    `shows:` `Scenario`, `Scenario.label`, `Scenario.screen`, `Trace.render`, `run_scenario`

    **An episode becomes a portable, pixel-free trace**

    *JSON, so it diffs in review and replays on a machine with nothing installed.*

    <img src="gallery/trace-record.png" width="860">

    `shows:` `Trace`, `Trace.blocked`, `Trace.choice`, `Trace.decided`, `Trace.decision_step`, `Trace.frames`, `Trace.load`, `Trace.name`, `Trace.record`, `Trace.save`, `Trace.to_dict`, `Trace.to_json`, `TraceStep`, `TraceStep.observation`, `TraceStep.to_dict`

    **Replay a recorded trace against a different brain**

    *The frames are the fixture; the brain is the only variable. Nothing is captured or clicked.*

    <img src="gallery/trace-replay.png" width="860">

    `shows:` `Trace.replay`, `Trace.observations`, `score_trace`

    **Counting how often the choice was right**

    *Scored on normalised text, so OCR spacing is forgiven - the same bargain the index makes.*

    <img src="gallery/score-trace.png" width="860">

    `shows:` `score_trace`, `score`, `score_replay`, `Verdict`, `Verdict.line`

    **Verdicts collected into an accuracy**

    *The number the whole module exists for: which brain is right, and how often.*

    <img src="gallery/scoreboard.png" width="860">

    `shows:` `Scoreboard`, `Scoreboard.accuracy`, `Scoreboard.accuracy_for`, `Scoreboard.add`, `Scoreboard.blocked`, `Scoreboard.brains`, `Scoreboard.by_brain`, `Scoreboard.confusion`, `Scoreboard.earned`, `Scoreboard.render`, `Scoreboard.to_rows`, `Scoreboard.total`

    **Every scenario through every brain, in one table**

    *A fresh brain per task, so state like 'already clicked this' cannot leak between them.*

    <img src="gallery/benchmark.png" width="860">

    `shows:` `benchmark`, `Scoreboard.to_rows`

    **Replay a whole recorded suite against one brain**

    *The literal roadmap step: the frames are fixed, nothing new is captured.*

    <img src="gallery/benchmark-traces.png" width="860">

    `shows:` `benchmark_traces`, `load_traces`, `Trace.save`

    ### Command line

    **--image example  (replay a screenshot, dry run)**

    *No display needed: the whole pipeline over a still image.*

    <img src="gallery/cli-image.png" width="860">

    `shows:` `main`, `build_parser`, `ImageScreen`, `detector_available`

    **--script  (replay a fixed action list instead of a brain)**

    *The script is validated exactly like a model's answer: nothing skips the gates.*

    <img src="gallery/cli-script.png" width="860">

    `shows:` `MockBrain`, `demo_screen`, `parse_action`

    **--verify-text  (the harness decides success, not the brain)**

    *Without it, an action that removes its own target looks like a failure.*

    <img src="gallery/cli-verify-text.png" width="860">

    `shows:` `AgentLoop.run`, `Episode.success`

    **--trace / --expected  (record the run, then score it)**

    *Writes JSON next to the code, and prints the score straight away.*

    <img src="gallery/cli-trace.png" width="860">

    `shows:` `Trace.record`, `Trace.save`, `score_trace`

    **--no-ocr  (index a numbered grid instead)**

    *Useful on a machine with no OCR engine, or for a canvas app.*

    <img src="gallery/cli-detector.png" width="860">

    `shows:` `GridDetector`, `make_detector`, `detector_available`

    **Exit codes: 0 success, 1 failure, 2 bad usage, 3 no screen**

    *A shell can tell 'it worked' from 'it gave up' without parsing any text.*

    <img src="gallery/cli-exit-codes.png" width="860">

    `shows:` `main`

    **The two deliberate acts before the mouse can move**

    *--real-input without the env var, and --brain vlm with no pixels: both refuse.*

    <img src="gallery/cli-refusals.png" width="860">

    `shows:` `main`, `ALLOW_REAL_INPUT_ENV`

    **--live with no display is refused, with the fix**

    *It tells you what to do instead of dying inside an episode.*

    <img src="gallery/cli-live-headless.png" width="860">

    `shows:` `display_available`, `main`

    **--live  (your real screen, dry run)**

    *Real capture, real OCR, a real element index - and not one click.*

    <img src="gallery/cli-live-dry-run.png" width="860">

    `shows:` `GrabScreen`, `IndexedScreen`, `AutoDetector`, `main`

    ### Real desktop

    **GrabScreen: capture a real desktop**

    *1280x800 of a real X server. mss first, Pillow's ImageGrab as the fallback.*

    <img src="gallery/desk-capture.png" width="860">

    `shows:` `GrabScreen`, `GrabScreen.capture`, `ScreenUnavailable`

    **The element index, drawn on the pixels it came from**

    *Every box is a real OCR word with a real screen coordinate: the model picks a label, not a pixel.*

    <img src="gallery/desk-ocr-index.png" width="860">

    `shows:` `OcrDetector`, `OcrDetector.detect`, `IndexedScreen`, `Element.center`

    **A real click, and the window's own reaction**

    *Pixels in, a label chosen, a pixel resolved, a button pressed - and the dialog closed itself.*

    <img src="gallery/desk-click.png" width="860">

    `shows:` `PyAutoGuiExecutor`, `AgentLoop.run`, `Policy.dry_run`, `resolve_point`

    **Success only when the harness can see it**

    *The app changed its own title and wrote a marker file; the verifier saw the change.*

    <img src="gallery/desk-verify.png" width="860">

    `shows:` `AgentLoop.run`, `Episode.success`, `Episode.stop_reason`

    **A screen with no text still has legal moves**

    *OCR finds nothing on a canvas, so the index degrades to numbered cells - and says so.*

    <img src="gallery/desk-grid-fallback.png" width="860">

    `shows:` `AutoDetector`, `GridDetector`, `OcrDetector`

    **The safety gate, on a real button, in a real window**

    *The Terminal button is visible and labelled - and the policy refuses to press it.*

    <img src="gallery/desk-policy-refusal.png" width="860">

    `shows:` `Policy.check`, `Policy.deny_labels`, `PolicyViolation`, `Observation.find`

    **The same click, driven end to end from the CLI**

    *--live --real-input --verify-text Saved: capture, OCR, click, verify, exit code.*

    <img src="gallery/desk-cli-real-input.png" width="860">

    `shows:` `main`, `PyAutoGuiExecutor`, `AgentLoop.run`

    ### Extensibility

    **Every stage is a Protocol: bring your own screen, brain, detector, executor**

    *The loop depends on four one-method contracts, not on the shipped implementations.*

    <img src="gallery/protocols.png" width="860">

    `shows:` `Brain`, `Brain.decide`, `ScreenSource`, `ScreenSource.capture`, `Detector`, `Detector.detect`, `Executor`, `Executor.execute`, `ActionResult`

    ## Why a tiny model can do this at all

    | Job | Who does it |
    |---|---|
    | Finding *where* a button is | OCR / accessibility tree — **not** the model |
    | Choosing *what* to do | the model, but only by picking a number from a short menu |
    | Knowing whether it worked | the verifier and the no-progress guard, not the model's word |
    | Recovering from a bad step | the retry policy in `loop.py`, not the weights |

    A 256M model cannot emit `(295, 284)` reliably, but it can pick "4" out of five options —
    and the harness owns the coordinate lookup, the vocabulary check and the definition of
    success.

    ## Safety, because this thing has your mouse

    - Run it in a **VM, container, or throwaway desktop account**, not your real session.
    - **Dry run is the default**, so you can read a whole episode before anything clicks.
    - Deny lists for typing (`sudo`, `rm -rf`, "password"), widgets (terminal, keychain,
      wallet, bank) and key chords.
    - A **kill-switch file** checked before every action, plus a hard step budget.
    - **Blind grid clicks are refused with real input** unless you pass `--allow-grid`: when
      OCR finds nothing, the fallback index is coordinates with no meaning, and a model
      choosing one is guessing about your desktop.
    - **A brain's "done" is a claim, not evidence**: pass `--verify-text` and the harness
      decides success itself.
    - **Local-first, for real:** pixels and keystrokes never leave the machine.

    ## What works, and what does not

    **Works:** real capture, a real OCR index with coordinates that land on the right pixel,
    a real click an app reacts to, verified success, the closed vocabulary, the policy gates,
    the no-progress guard, echo detection, the doctor, the CLI, portable traces that score a brain's choices offline, skills learned from verified runs,
    a resumable task runner, operator approval gates, a watchdog that refuses to wait forever rather than stalling silently, a hash-chained audit log, 327 tests, a 256M VLM that
    reads a button label from pixels on a clean screenshot — and the same suite passing inside
    a real Windows Server 2022 guest with no KVM (109 passed, exit 0, with the OCR-inference
    tests excluded as impractically slow under emulation).

    **Does not work yet:** reliability on a *live* screen. The 256M VLM echoed the prompt
    there; the 0.5B text planner picks wrong actions; UI Automation has never been exercised
    against a real Windows desktop, so an agent there still reads labels from OCR; and the
    harness still trusts a model's `done` when no verifier is given.

    ## Roadmap

    - [x] Capture, OCR index, closed vocabulary, dry-run default, policy gates, doctor
    - [x] A real click on a real window, with a verified outcome
    - [x] Blind-grid-click guard, echo detection, no-progress guard
    - [x] The harness suite running on real Windows (Server 2022 guest, no KVM — see above)
    - [x] Score episodes: replay a trace and count how often the choice was right - see [scoring a brain, offline](#scoring-a-brain-offline)
    - [x] A capability gallery: one real screenshot per capability, generated by `showcase.py`
    - [x] The OCR guard skips instead of failing: one tested guard, `conftest.require_ocr`
    - [x] Stop the 256M VLM echoing the prompt - masked decoding, measured in
          [asking a tiny model in four ways](#asking-a-tiny-model-in-four-ways-measured)
    - [x] The marimo console: live screen preview, action log, pause / resume / step
    - [x] An accessibility-tree detector on Linux (AT-SPI), driving a real click with no OCR
    - [x] Make a tiny model choose correctly: a 0.5B text planner over the accessibility index in
          `rank` mode gets **4/4** goals right in 0.9s. A 256M **vision** model does not (1-2/4) -
          it is below the threshold, which is why the rule-free planner is still the default
    - [x] A daily-usage evaluation: 11 tasks (menu, scroll, typing, shortcut, second window,
          verify, refuse, kill switch) against a real GTK app, checked against the application's
          own log - see [can it do a day's work](#can-it-do-a-days-work)
    - [x] **2.0** Skills: a verified run is written down once and replayed with no model at all
    - [x] **2.0** Operator approval gates - a label whitelist, or ask before every action
    - [x] **2.0** A resumable task queue with a report, and structural verification of "it worked"
    - [x] **2.0** `skills` and `batch` subcommands, and an example queue at `tasks/day.json`
    - [x] **2.0** A watchdog: a per-call and whole-episode deadline, so a call that never returns
          becomes a named step instead of a console that looks like it is thinking
    - [x] **2.0** An audit log: an append-only, hash-chained record of a run, and `--verify-audit`
          to check it - so the evidence survives the container that produced it
    - [x] **2.0** `--gate`, `--audit` and `--verify-audit` on the command line, and the shared
          AT-SPI guard, so a machine without the typelib skips rather than failing
    - [ ] A gallery run that pictures the 2.0 additions: **107 of the package's 270 public callables**
          have no screenshot yet - 103 in the operator layer and four CLI entry points. Adding the
          entries to `showcase_catalog.py` is what closes it
    - [ ] **Verify UI Automation** (Windows) against a real Windows desktop. The reader and the
          detector are implemented and unit-tested against a synthetic control tree, and
          `accessibility_available("uia")` reports the truth on any machine - but no Windows
          has run the real backend yet, which is the same GitHub-runner blocker as below.
    - [ ] Windows run of the *live desktop* test: the workflow is committed, but it needs a
          GitHub mirror with Actions enabled — Hugging Face does not run GitHub Actions
    - [ ] A **Windows** run of the gallery: the workflow builds it on `windows-latest` (job
          `gallery-windows`), with the same GitHub-mirror caveat - and when it runs, its
          artifact still has to be imported into `gallery/` as `win-*` and spliced in here;
          no step does that yet

    ## License

    MIT — see [LICENSE](LICENSE).
    '''

    mo.md(readme_md)
    return (readme_md,)


@app.cell(hide_code=True)
def repo_files():
    pyproject_toml = '''
    [build-system]
    requires = ["setuptools>=68"]
    build-backend = "setuptools.build_meta"

    [project]
    name = "tiny-computer-agent"
    version = "2.0.0"
    description = "An agentic harness that drives a real computer with a tiny, fully local model - no cloud API. It learns from verified runs."
    readme = "README.md"
    requires-python = ">=3.10"
    license = { text = "MIT" }
    authors = [{ name = "hkblockchaintechnology" }]
    keywords = ["computer-use", "gui-agent", "agentic-harness", "local-first", "tiny-models"]
    classifiers = [
      "Development Status :: 3 - Alpha",
      "Intended Audience :: Developers",
      "Programming Language :: Python :: 3 :: Only",
      "Topic :: Scientific/Engineering :: Artificial Intelligence",
    ]
    dependencies = []   # the harness core imports nothing outside the standard library

    [project.optional-dependencies]
    ocr = ["rapidocr-onnxruntime>=1.3"]        # labels from pixels, no system binary
    desktop = ["mss>=9.0", "pyautogui>=0.9", "pillow>=10"]   # capture + real input
    vision = ["transformers>=4.48", "torch>=2.2", "pillow>=10"]  # tiny model brains
    dev = ["pytest>=8"]

    [project.scripts]
    tiny-computer-agent = "tiny_computer_agent.__main__:main"

    [project.urls]
    Homepage = "https://huggingface.co/hkblockchaintechnology/tiny-computer-agent"

    [tool.setuptools]
    packages = ["tiny_computer_agent"]

    [tool.pytest.ini_options]
    testpaths = ["tests"]
    '''

    license_txt = '''MIT License

    Copyright (c) 2026 hkblockchaintechnology

    Permission is hereby granted, free of charge, to any person obtaining a copy
    of this software and associated documentation files (the "Software"), to deal
    in the Software without restriction, including without limitation the rights
    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    copies of the Software, and to permit persons to whom the Software is
    furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in all
    copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    SOFTWARE.
    '''

    gitignore_txt = '''__pycache__/
    *.py[cod]
    .venv/
    venv/
    .env
    *.gguf
    models/
    runs/
    traces/
    .ipynb_checkpoints/
    .DS_Store
    '''

    mo.md(f"packaging: {len(pyproject_toml) + len(license_txt) + len(gitignore_txt)} chars, "
          f"v2.0.0 with ocr/desktop/vision extras")
    return gitignore_txt, license_txt, pyproject_toml


@app.cell
def publish():
    REPO_ID = "hkblockchaintechnology/tiny-computer-agent"

    publish_button = mo.ui.button(
        label="⬆️ Publish README.md to the Hub",
        value=0,
        on_click=lambda value: value + 1,
    )

    publish_button
    return REPO_ID, publish_button


@app.cell
def publish_status(
    REPO_ID,
    gitignore_txt,
    license_txt,
    publish_button,
    pyproject_toml,
    readme_md,
):
    _status = mo.md(
        "This notebook owns the project: the card, the packaging files, the package and "
        "the tests. Click **Publish** to push all of it to the Hub."
    ).callout(kind="info")

    if publish_button.value:
        _token = os.environ.get("HF_TOKEN")
        if not _token:
            _status = mo.md(
                "No `HF_TOKEN` in the environment. Run `export HF_TOKEN=...` "
                "(or `hf auth login`), then restart the kernel."
            ).callout(kind="warn")
        else:
            _api = HfApi(token=_token)
            _root = Path(__file__).parent
            _notebook = Path(__file__)
            _files = {
                "README.md": readme_md,
                "pyproject.toml": pyproject_toml,
                "LICENSE": license_txt,
                ".gitignore": gitignore_txt,
                "tiny_computer_agent_app.py": _notebook.read_text(encoding="utf-8"),
            }
            # the tool that builds the capability gallery, shipped with the card it fills
            for _tool in ("showcase.py", "showcase_core.py", "showcase_catalog.py",
                          "measure_vlm.py", "evaluate_daily.py"):
                if (_root / _tool).is_file():
                    _files[_tool] = (_root / _tool).read_text(encoding="utf-8")
            _folders = [f for f in ("tiny_computer_agent", "tests", "gallery",
                                    "tasks", "showcase_scenes", "preview")
                        if (_root / f).is_dir()]
            if (_root / ".github").is_dir():
                _folders.append(".github")          # the CI that builds the Windows gallery
            _missing = [f for f in ("tiny_computer_agent", "tests") if not (_root / f).is_dir()]
            if _missing:
                _status = mo.md(f"Nothing to publish: missing {_missing} next to `{_root}`").callout(kind="danger")
            else:
                try:
                    for _name, _text in _files.items():
                        _api.upload_file(
                            path_or_fileobj=_text.encode("utf-8"),
                            path_in_repo=_name,
                            repo_id=REPO_ID,
                            repo_type="model",
                            commit_message=f"Publish {_name} from marimo",
                        )
                    for _folder in _folders:
                        _api.upload_folder(
                            folder_path=str(_root / _folder),
                            path_in_repo=_folder,
                            repo_id=REPO_ID,
                            repo_type="model",
                            commit_message=f"Publish {_folder}/ from marimo",
                            ignore_patterns=["__pycache__/*", "*.pyc"],
                        )
                    _status = mo.md(
                        f"✅ Published {len(_files)} files and {len(_folders)} folders to "
                        f"[{REPO_ID}](https://huggingface.co/{REPO_ID})"
                    ).callout(kind="success")
                except Exception as _e:
                    _status = mo.md(f"Publish failed: `{type(_e).__name__}: {_e}`").callout(kind="danger")

    _status
    return


@app.cell
def card_preview(readme_md):
    # The published model card, as it renders on the Hub right now.
    import json

    from PIL import Image

    _card_root = Path(__file__).parent          # Path comes from the setup cell
    _gallery_index = json.loads((_card_root / "gallery" / "index.json").read_text())
    _preview_shots = sorted((_card_root / "preview").glob("0*.png"))

    card_status = mo.md(
        f"**The published card.** {len(readme_md):,} characters, "
        f"{readme_md.count('<img src="gallery/')} screenshots covering "
        f"{len(_gallery_index)} capabilities - every public callable pictured. "
        f"Live at [hkblockchaintechnology/tiny-computer-agent]"
        f"(https://huggingface.co/hkblockchaintechnology/tiny-computer-agent)."
    )

    mo.vstack(
        [card_status]
        + (
            [mo.image(Image.open(shot)) for shot in _preview_shots]
            if _preview_shots
            else [
                mo.md(
                    "_No preview captured yet - run `python preview/capture.py` to "
                    "screenshot the live page._"
                )
            ]
        )
    )
    return


@app.cell
def demo():
    # A real screenshot -> real OCR -> labelled, clickable elements. No display needed:
    # this is the same path --image takes, and it is what makes a tiny model viable.
    _detector = OcrDetector()
    _observation = IndexedScreen(ImageScreen(fake_desktop()), _detector).capture()

    perception_rows = [
        {"label": e.label, "kind": e.kind, "x": e.x, "y": e.y, "w": e.width, "h": e.height}
        for e in _observation.elements
    ]

    perception_episode = AgentLoop(
        screen=IndexedScreen(ImageScreen(fake_desktop()), _detector),
        brain=PlannerBrain(),
        executor=DryRunExecutor(),
        policy=Policy(step_budget=4),
    ).run("click \'Save\' to save the document")

    _clicked = next((s.result.note for s in perception_episode.steps if s.action and s.action.kind == "click"), "n/a")

    mo.vstack(
        [
            mo.md(
                f"**Perception, from pixels.** OCR read **{len(perception_rows)} labels** off a "
                f"synthetic screenshot; the planner then chose one and the harness resolved it to a real pixel."
            ),
            mo.ui.table(perception_rows, selection=None),
            mo.md(f"`{_clicked}`"),
            mo.md(f"`{perception_episode.render().splitlines()[1]}`"),
        ]
    )
    return


@app.cell
def operator_layer():
    # The 2.0 operator layer, exercised rather than described: a gate that consents per action,
    # a watchdog that refuses to wait, and an audit log that can be checked afterwards.
    import json as _json
    import tempfile as _tempfile

    _goal = "click 'Save' to save the document"


    def _room():
        """A two-button screen, so the demo is about consent and evidence, not about OCR."""
        return FakeScreen([(Element("Save", x=320, y=120, width=80, height=24),
                            Element("Cancel", x=440, y=120, width=80, height=24))])


    class _Hung(DryRunExecutor):
        """An executor that never returns - the failure a step budget cannot catch."""

        def execute(self, action, observation):
            import time

            time.sleep(3)
            return super().execute(action, observation)


    _audit_path = Path(_tempfile.mkdtemp()) / "run.jsonl"
    _audit = AuditLog(_audit_path)
    _watchdog = Watchdog(step_timeout=30, wall_clock=120)

    # 1. an ordinary episode, with a gate, a deadline and a record around it
    _episode = AgentLoop(
        screen=_room(), brain=PlannerBrain(), executor=DryRunExecutor(),
        policy=Policy(step_budget=3), gate=gate_from_spec("labels:Save"),
        audit=_audit, watchdog=_watchdog,
    ).run(_goal)

    # 2. the same goal behind a read-only gate: the operator's no is a step, not a crash
    _refused = AgentLoop(screen=_room(), brain=PlannerBrain(), executor=DryRunExecutor(),
                         policy=Policy(step_budget=3),
                         gate=gate_from_spec("read-only")).run(_goal)

    # 3. a call that hangs: without the watchdog this cell would never finish
    _hung = AgentLoop(screen=_room(), brain=MockBrain([{"kind": "click", "label": "Save"}]),
                      executor=_Hung(), watchdog=Watchdog(step_timeout=0.05)).run(_goal)

    # 4. the record: check it, then tamper with a copy and let the verifier name the entry
    _lines = _audit_path.read_text().splitlines()
    _entry = _json.loads(_lines[0])
    _entry["outcome"] = "nothing happened at all"           # a nicer story than the truth
    _tampered = _audit_path.with_name("tampered.jsonl")
    _tampered.write_text(_json.dumps(_entry, sort_keys=True, separators=(",", ":")) + "\n"
                         + "\n".join(_lines[1:]) + "\n")

    operator_rows = [
        {"2.0 piece": "gate: labels:Save", "what happened": "let the whitelisted click through"},
        {"2.0 piece": "gate: read-only", "what happened": f"refused - {_refused.steps[-1].error}"},
        {"2.0 piece": "watchdog", "what happened": _hung.steps[-1].error},
        {"2.0 piece": "audit: intact", "what happened": verify_audit(_audit_path)[1]},
        {"2.0 piece": "audit: edited", "what happened": verify_audit(_tampered)[1]},
    ]

    operator_layer = mo.vstack([
        mo.md(f"**The 2.0 operator layer.** One goal - `{_goal}` - three ways: consented, refused, "
              f"and hung. {_watchdog.render()}"),
        mo.ui.table(operator_rows, selection=None),
        mo.md("```\n" + _episode.render() + "\n```"),
    ])
    operator_layer

    return


@app.cell
def scoreboard():
    # The brains are separable, so they can be scored against the same tasks with no
    # display, no OCR and no weights: a scenario is an element index plus the answer that
    # is right. Record a run once; replay it against any brain, and count how often right.


    class FirstItemBrain:
        """A baseline with no model at all: click whatever is first in reading order."""

        name = "first-item"

        def decide(self, goal, observation):
            menu = DecisionMenu(observation)
            if not menu:
                return Action("done", reason="nothing to click")
            return Action("click", label=menu.items[0].label, reason="first item in reading order")


    def _bar(label, x):
        return Element(label, x=x, y=6, width=80, height=20)


    def _button(label, x, y=270):
        return Element(label, x=x, y=y, width=90, height=30)


    score_scenarios = [
        Scenario(
            "click 'Save' to save the document",
            frames=[
                (_bar("Files", 0), _bar("Edit", 100), _bar("View", 200)),
                (Element("Name", x=200, y=195, width=200, height=26), _button("Save", 200)),
                (_bar("Files", 0), _button("Save", 200)),
            ],
            expected="Save", name="save the document",
        ),
        Scenario(
            "click 'Files' to open the file manager",
            frames=[(_bar("Files", 0), _bar("Edit", 100), _button("Save", 200))],
            expected="Files", name="open the file manager",
        ),
        Scenario(
            "click 'Cancel' to discard the change",
            frames=[(_button("Save", 200), _button("Cancel", 400))],
            expected="Cancel", name="discard the change",
        ),
    ]

    scoreboard = benchmark(
        score_scenarios,
        {
            "planner (rule-based)": lambda scenario: PlannerBrain(),
            "first-item (baseline)": lambda scenario: FirstItemBrain(),
            "oracle (always right)": lambda scenario: MockBrain(
                [
                    {"kind": "click", "label": scenario.expected},
                    {"kind": "done", "reason": "clicked the answer the task names"},
                ]
            ),
        },
    )

    mo.vstack(
        [
            mo.md(
                f"**Score the choice, offline.** {len(score_scenarios)} tasks x "
                f"{len(scoreboard.brains())} brains = **{scoreboard.total} decisions**, "
                f"**{scoreboard.earned} of them right ({scoreboard.accuracy:.0%})** - read off "
                f"saved element indexes, with no screen, no model and no network."
            ),
            mo.ui.table(scoreboard.to_rows(), selection=None),
            mo.md(
                "Accuracy per brain: "
                + " · ".join(
                    f"**{_brain}** {_accuracy:.0%}"
                    for _brain, _accuracy in scoreboard.by_brain().items()
                )
            ),
        ]
    )
    return


@app.cell
def console_setup():
    # The console's controls. This cell owns the mutable session, so the console below
    # can re-run on every click without ever losing the episode in progress.
    import tempfile

    console_goal = mo.ui.text(value="click 'Save' to save the document", label="goal",
                              full_width=True)
    console_brain = mo.ui.dropdown(
        options={"planner (rule-based, no weights)": "planner", "scripted clicks": "scripted"},
        value="planner (rule-based, no weights)", label="brain")
    console_screen = mo.ui.dropdown(
        options={"replayed screenshot + real OCR": "image", "this machine, live": "live"},
        value="replayed screenshot + real OCR", label="screen")
    console_interval = mo.ui.refresh(options=[5, 10, 30], default_interval=10,
                                     label="live preview")
    console_step = mo.ui.run_button(label="▶ step")
    console_run = mo.ui.run_button(label="⏩ run", kind="success")
    console_reset = mo.ui.run_button(label="⟲ reset", kind="warn")
    console_pause = mo.ui.switch(label="pause", value=False)
    console_real_input = mo.ui.switch(label="real mouse (needs the allow env var)", value=False)
    console_panic = mo.ui.switch(label="panic: create the kill switch", value=False)

    console_session = {
        "loop": None,
        "signature": None,
        "kill": str(Path(tempfile.gettempdir()) / "tiny-computer-agent.console.STOP"),
    }
    return (
        console_brain,
        console_goal,
        console_interval,
        console_panic,
        console_pause,
        console_real_input,
        console_reset,
        console_run,
        console_screen,
        console_session,
        console_step,
    )


@app.cell
def console(
    console_brain,
    console_goal,
    console_interval,
    console_panic,
    console_pause,
    console_real_input,
    console_reset,
    console_run,
    console_screen,
    console_session,
    console_step,
):
    # The console: live screen preview, the action log, and step / pause / resume.
    from PIL import ImageDraw

    _ALLOW_ENV = "TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT"
    _goal = console_goal.value.strip() or "click 'Save' to save the document"
    _signature = (_goal, console_brain.value, console_screen.value,
                  bool(console_real_input.value))
    _session = console_session
    _loop = _session["loop"]

    # the panic switch drives the same kill-switch file the CLI checks
    if console_panic.value:
        Path(_session["kill"]).write_text("stop")
    else:
        Path(_session["kill"]).unlink(missing_ok=True)

    if _loop is None or _session["signature"] != _signature:
        if console_brain.value == "scripted clicks":
            _brain = MockBrain([{"kind": "click", "label": "Save"},
                                {"kind": "done", "reason": "clicked Save"}])
        else:
            _brain = PlannerBrain()
        if console_screen.value == "this machine, live":
            _screen = IndexedScreen(GrabScreen(monitor=1), OcrDetector())
        else:
            _screen = IndexedScreen(ImageScreen(fake_desktop()), OcrDetector())
        _real = bool(console_real_input.value) and os.environ.get(_ALLOW_ENV) == "1"
        _loop = AgentLoop(
            screen=_screen, brain=_brain,
            executor=PyAutoGuiExecutor() if _real else DryRunExecutor(),
            policy=Policy(dry_run=not _real, step_budget=8, kill_switch_path=_session["kill"]),
        )
        _loop.begin(_goal)
        _session.update(loop=_loop, signature=_signature)

    # the controls: run_button is True only on the run its own click started, so an
    # auto-refresh can never fire a step by accident
    _loop.paused = bool(console_pause.value)
    if console_reset.value:
        _loop.begin(_goal)
    if console_step.value:
        _loop.paused = False          # a single step is allowed through a pause
        _loop.step_once()
    if console_run.value:
        _loop.run(_goal, max_steps=8)  # respects the pause switch, and the goal

    # a live peek at the screen, without taking a turn
    _observation = _loop.observation
    if console_screen.value == "this machine, live":
        _observation = _loop.screen.capture()
        _armed = console_interval.value          # referencing it arms auto-refresh

    _preview = None
    if _observation is not None and hasattr(_observation.frame, "convert"):
        _preview = _observation.frame.convert("RGB").copy()
        _pen = ImageDraw.Draw(_preview)
        for _element in _observation.elements:
            _pen.rectangle(
                [_element.x, _element.y,
                 _element.x + max(_element.width, 1), _element.y + max(_element.height, 1)],
                outline=(255, 138, 0), width=2)
            _pen.text((_element.x + 3, max(0, _element.y - 12)), _element.label[:30],
                      fill=(210, 90, 0))

    _rows = [{"step": _step.index,
              "action": _step.action.describe() if _step.action else "",
              "outcome": _step.error or (_step.result.summary() if _step.result else "")}
             for _step in _loop.episode.steps]

    _state = "finished" if _loop.finished else ("paused" if _loop.paused else "ready")
    _mode = "REAL mouse" if not _loop.executor.dry_run else "dry run (nothing is clicked)"
    console_status = mo.md(
        f"**{_state}** · {len(_rows)} step(s) · {_mode} · budget {_loop.policy.step_budget} · "
        f"stop: `{_loop.episode.stop_reason}`"
    )
    if console_real_input.value and os.environ.get(_ALLOW_ENV) != "1":
        console_status = mo.vstack([
            console_status,
            mo.md(f"`{_ALLOW_ENV}` is not set, so this stays a dry run.").callout(kind="warn"),
        ])

    mo.vstack([
        mo.hstack([console_goal, console_brain, console_screen, console_interval],
                  justify="start", wrap=True),
        mo.hstack([console_step, console_run, console_pause, console_reset,
                   console_real_input, console_panic], justify="start", wrap=True),
        console_status,
        mo.image(_preview) if _preview is not None else mo.md("_no frame yet_"),
        mo.ui.table(_rows, selection=None) if _rows else mo.md("_no actions yet_"),
    ])
    return


if __name__ == "__main__":
    app.run()