| |
| |
| |
| |
| |
| |
| |
|
|
| import marimo |
|
|
| __generated_with = "0.25.2" |
| app = marimo.App(width="medium", auto_download=["html"]) |
|
|
| with app.setup: |
| import os |
| from pathlib import Path |
|
|
| import marimo as mo |
| from huggingface_hub import HfApi |
|
|
| |
| from tiny_computer_agent import ( |
| AgentLoop, |
| DryRunExecutor, |
| Element, |
| GrabScreen, |
| MockBrain, |
| PlannerBrain, |
| Policy, |
| PyAutoGuiExecutor, |
| ) |
| from tiny_computer_agent.actions import Action |
| from tiny_computer_agent.brain import DecisionMenu |
| from tiny_computer_agent.demo import fake_desktop |
| from tiny_computer_agent.observe import ImageScreen |
| from tiny_computer_agent.perception import IndexedScreen, OcrDetector |
| from tiny_computer_agent.trace import Scenario, benchmark |
|
|
|
|
| @app.cell(hide_code=True) |
| def readme(): |
| readme_md = r''' |
| |
| --- |
| title: Tiny Computer Agent |
| emoji: 🖱️ |
| colorFrom: indigo |
| colorTo: green |
| license: mit |
| tags: |
| - computer-use |
| - gui-agent |
| - agentic-harness |
| - local-first |
| - tiny-models |
| - offline |
| - marimo |
| --- |
| |
| # 🖱️ Tiny Computer Agent |
| |
| **An agentic harness that drives a real computer with a tiny, fully local model.** |
| No cloud API, no API keys, no telemetry — and a core that imports nothing outside the |
| Python standard library. |
| |
| > **Status 2.0.0: it drives a real desktop — on Linux and on Windows — and it learns.** |
| > Screen capture, an OCR element index with real coordinates, a real mouse click, and a |
| > *verified* outcome. A verified run can be written down as a **skill** and replayed with no |
| > model at all; a queue of goals can be run, resumed after a kill, and reported on; an |
| > **approval gate** can stand in front of every action; and what changed on screen is a |
| > **measured** structural diff rather than a guess. |
| > **All 74 capabilities have a real screenshot** in the [capability gallery](#capability-gallery) (Linux captures). |
| > The loop can be driven from a [marimo console](#the-marimo-console) with a live preview, an |
| > action log and pause / resume / step; elements can come from the |
| > [accessibility tree](#what-the-machine-already-knows-the-accessibility-tree) instead of OCR; |
| > and a run can be saved as a portable *trace*, replayed offline and **scored**, so |
| > "which brain is right, and how often?" is a number in a table. |
| > The suite passes on a live Linux desktop and inside a real Windows Server |
| > 2022 guest; what is not solved is model reliability — see |
| > [measured results](#measured-results). |
| |
| ## The idea, in one loop |
| |
| ``` |
| observe think act verify |
| +---------+ +---------+ +---------+ +---------+ |
| | screen |--->| tiny |--->| mouse / |--->| next | |
| | frame | | model | | keyboard| | frame | |
| +---------+ +---------+ +---------+ +----+----+ |
| | |
| <-------------------------------------------------+ |
| repeat until done, or the step budget runs out |
| ``` |
| |
| 1. **Observe** — capture the screen, then build an **element index**: OCR (and, |
| optionally, the OS accessibility tree) locates text and widgets and gives each one a |
| label and a real screen coordinate. |
| 2. **Think** — the model is shown a *short numbered menu* built from that index and |
| answers with one number (plus a verb). It never emits coordinates. |
| 3. **Act** — the harness maps the chosen label back to pixels and executes it. |
| 4. **Verify** — re-read the screen; stop if the same action repeats against an unchanged |
| screen, and only call the episode a success if a verifier can see it worked. |
| |
| ## Measured results |
| |
| Everything below was run from this repo on CPU, dry-run unless stated. Nothing is |
| aspirational. |
| |
| ### On a live desktop, with a real click |
| |
| A real Tk window (a Save dialog with Save/Cancel buttons) on a **real X server** |
| (`Xvfb :99`), driven with real mouse input. The button sits at window `+120+120`, |
| `x=20, y=155`, `110x40`, so its centre is `(195, 295)`: |
| |
| ``` |
| $ python -m tiny_computer_agent --live --real-input --verify-text Saved "click 'Save' to save the document" |
| mode: REAL INPUT (mouse and keyboard live) | screen: real desktop (monitor 1) | brain: planner |
| index: auto - OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline)) |
| |
| step 0 | click label='Save' reason="matched 'Save' in the element index" | ok: clicked (195, 296) |
| |
| result: success - verified after 1 action(s) |
| ``` |
| |
| The click landed on the button, the app changed its own window to "Saved", and the |
| verifier confirmed it before the episode was called a success. **Pixels in, a real click |
| out, and evidence that it worked.** |
| |
| ### Re-confirmed end to end on a headless Linux box |
| |
| The whole path above was re-run from scratch on a headless Debian container (gVisor |
| sandbox, 20 vCPU, no GPU, no physical display, no network at run time) behind a virtual |
| X server. First the suite, headless: |
| |
| ```bash |
| python -m pip install -e ".[desktop,ocr,dev]" |
| pytest tests -q |
| # 227 passed, 2 skipped in 18.7s (that run was 0.3.0; 2.0 has grown this to 284) |
| ``` |
| |
| The two skips are the live-desktop tests. Give them a display and they run for real: |
| |
| ```bash |
| Xvfb :99 -screen 0 1280x800x24 & |
| export DISPLAY=:99 |
| TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py -q |
| # tests/test_live_desktop.py::test_ocr_finds_a_real_button_at_the_right_pixels PASSED |
| # tests/test_live_desktop.py::test_a_real_click_lands_on_the_button PASSED |
| # 2 passed |
| ``` |
| |
| Then the same desktop, driven through the CLI with the mouse actually live: |
| |
| ```bash |
| export TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1 |
| python -m tiny_computer_agent --live --real-input --verify-text Saved \ |
| "click 'Save' to save the document" |
| ``` |
| |
| ``` |
| goal: click 'Save' to save the document |
| mode: REAL INPUT (mouse and keyboard live) | screen: real desktop (monitor 1) | brain: planner | kill switch: create /tmp/tiny-computer-agent.STOP to stop |
| index: auto - OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline)) |
| |
| step 0 | click label='Save' reason="matched 'Save' in the element index" | ok: clicked (194, 296) |
| |
| result: success - verified after 1 action(s) |
| ``` |
| |
| The window was drawn at `+120+120` with its Save button at `x=20, y=155, 110x40`, so the |
| button centre is `(195, 295)`. The click at `(194, 296)` is *on the button*, not near it — |
| and the app wrote `Save` back to its own marker file. Pixels in, a real click out, and the |
| app's own side effect as the proof. |
| |
| `--doctor` on that box, with the virtual display attached: |
| |
| ``` |
| tiny-computer-agent preflight (Linux x86_64, python 3.13.11) |
| |
| [ok ] display server: DISPLAY=:99 |
| [ok ] pillow: installed |
| [ok ] screen capture: mss (fast, multi-monitor) |
| [ok ] input backend (pyautogui): installed |
| [ok ] element detector (auto): OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline)) |
| [ok ] model runtime: not needed: the planner is rule-based (no weights, no download) |
| [ok ] execution mode: dry run (nothing is clicked) |
| [ok ] kill switch: create /tmp/tiny-computer-agent.STOP to abort the next action |
| |
| READY: the harness can drive this desktop. |
| ``` |
| |
| |
| ### On a replayed screenshot |
| |
| ``` |
| $ python -m tiny_computer_agent --image example --brain planner |
| elements detected: 5 |
| step 0 | click label='Save' ... -> pixel (295, 284) # OCR found the button |
| ``` |
| |
| **SmolVLM-256M-Instruct** (256M parameters, CPU) read the button off the pixels and |
| named the right target: |
| |
| ``` |
| step 0 | click label='Save' reason="the reply names 'Save'" -> pixel (295, 284) |
| ``` |
| |
| ### Scoring a brain, offline |
| |
| "Which brain is right, and how often?" needs no desktop, no model and no network, |
| because a **trace** keeps every frame's element index and throws the pixels away. Record a |
| run once; replay it against any brain, as often as you like. |
| |
| ``` |
| $ python -m tiny_computer_agent --image example --trace runs/save.json --expected Save |
| ... |
| trace: wrote runs/save.json (2 frames) |
| score: correct click 'Save' to save the document: expected 'Save', chose 'Save' |
| ``` |
| |
| A trace is JSON, so it diffs in review, commits next to the tests, and replays on a machine |
| with nothing installed. Scoring a whole suite is a table: |
| |
| ``` |
| scoreboard: 7/9 correct (78%) |
| planner (rule-based): 100% |
| first-item (baseline): 33% |
| oracle (always right): 100% |
| correct save the document: expected 'Save', chose 'Save' |
| incorrect save the document: expected 'Save', chose 'Files' (chose 'Files', not 'Save') |
| ... |
| ``` |
| |
| The number is the brain's **decision** - what it chose when it had to commit. Whether the |
| harness *allowed* that decision to run is a separate axis: a click the policy refused is |
| still the choice it was, and it is flagged `(harness refused it)` rather than passed off as |
| a clean success. Keeping those two apart is the point. |
| |
| ### Asking a tiny model in four ways (measured) |
| |
| The card used to say the 256M VLM "echoes the prompt instead of choosing". That is |
| reproducible - and so is the fix. A live GTK window, a menu of four exact widgets |
| (Files, Terminal, Save, Cancel), and the goal `click 'Save' to save the document`: |
| |
| ``` |
| mode echoed seconds outcome |
| json no 8.9 BrainError: the reply mentions 4 menu entries, so it looks like an echo |
| choice no 1.9 BrainError: the constrained reply was not a menu number: '1000' |
| scan no 1.4 click label='Files' (the model said yes to the first candidate) |
| rank no 9.4 click label='Files' (highest yes/no log-odds margin, +0.94) |
| ``` |
| |
| Four ways to ask, all of them the same weights: |
| |
| | mode | how the model is asked | answer space | |
| |---|---|---| |
| | `json` | reply with JSON, parsed leniently | the whole vocabulary | |
| | `choice` | one masked generation for a menu number | a handful of digits | |
| | `scan` | one masked yes/no question per item, first yes wins | two tokens | |
| | `rank` | the same questions, ranked by the yes/no log-odds margin | two tokens | |
| |
| What the numbers actually say: |
| |
| - **The echo is fixed, structurally.** In `json` mode the model hands the prompt |
| back (`"The goal is to click 'Save' to save the document..."`) and the harness |
| refuses it. In the three masked modes the reply *cannot* be the prompt, because |
| every one of its tokens is masked out of the logits. The failure stopped being |
| *detected* and started being *not representable*. |
| - **It is about six times faster** - 8.9 s of free text becomes 1.4 s for a |
| one-token answer - because the model is no longer asked to write. |
| - **Accuracy is not fixed, and the numbers say why.** The model answers now, and |
| answers *wrongly*: it accepts `Files` for a goal that names `Save`, even when the |
| answer space is two tokens and each candidate is judged on its own. A 256M model |
| cannot do this task. That is a finding about the weights, not about the harness - |
| which is why the rule-based planner is still the default brain. |
| |
| Reproduce it with `python measure_vlm.py --screen live --detector best --open-gtk`. |
| |
| ### Which brain drove which run |
| |
| The harness and the brain are separable, and it matters which one a result came from. |
| None of the runs below used a cloud API; the two model brains run on CPU: |
| |
| | Run | Brain | Weights | Outcome | |
| |---|---|---|---| |
| | Live Linux desktop, real click | `PlannerBrain` (rule-based) | none | **success, verified** | |
| | `/tmp/saveapp.py` window live | `PlannerBrain` | none | clicked `Save` at `(194, 296)` | |
| | `--image example` | `PlannerBrain` | none | picked `Save`, pixel `(295, 284)` | |
| | `--image example` | `VLMBrain` (SmolVLM-256M-Instruct) | 256M, CPU | picked `Save` | |
| | Live Linux desktop | `VLMBrain` (SmolVLM-256M-Instruct) | 256M, CPU | echoed the prompt; **refused** by echo detection | |
| | Live Linux desktop | `TextPlannerBrain` (Qwen2.5-0.5B) | 0.5B, CPU | clicked Cancel, then garbled JSON; budget exhausted | |
| |
| The rule-based planner is what carries the verified end-to-end runs. The tiny model is the |
| open problem: it is reliable on a clean screenshot and unreliable on a busy live screen. |
| |
| ### What did *not* work, and what caught it |
| |
| - **The 256M VLM echoes the prompt on a busy live screen.** Asked to click Save on the |
| live Tk window it replied with the prompt text back. The harness's echo detection |
| refused it (`BrainError: the reply mentions 4 menu entries, so it looks like an echo of |
| the prompt`) and stopped — **no blind click**. On the cleaner synthetic screenshot the |
| same model chose correctly, so this is a prompt/robustness problem, not a wiring one. |
| - **Qwen2.5-0.5B text planner over the OCR index is not reliable**: it clicked Cancel |
| before Save, then recovered a garbled JSON reply, then found Save — budget exhausted. |
| - **The grid fallback announced itself.** After the dialog closed, OCR found no text, the |
| index degraded to 96 numbered cells, and the run printed |
| `warning: no text was found on screen, so the index is 96 numbered grid cells and the |
| agent is choosing blind` — instead of silently guessing. |
| |
| **284 tests pass**, including real OCR over a real image and (opt-in) live-desktop tests |
| that click a real button. |
| |
| ## Verified on real Windows |
| |
| The Windows claims in this card are not taken on faith. A full Windows guest was built |
| and the harness installed inside it, on a headless Linux host with **no KVM** — the same |
| kind of sandbox the Linux runs above were produced in. |
| |
| > **Provenance.** The numbers below are quoted from that run, not re-measured here, and no |
| > picture from it survives: the guest, its disk image and its screenshots went with the |
| > container that had produced them. Everything above - the gallery, the live click, the |
| > traces - is reproducible from this repository; the Windows results below are a record of |
| > what that guest reported. |
| |
| **What was built.** |
| |
| ``` |
| host headless Linux container (no /dev/kvm; software emulation only) |
| hypervisor QEMU 10.0.13 -accel tcg,thread=multi -m 8G -smp 8 -cpu Skylake-Client |
| firmware SeaBIOS (the OVMF/UEFI build would not auto-boot the installation media) |
| guest Windows Server 2022 Standard Evaluation (Desktop Experience), x86_64 |
| install unattended from the public evaluation ISO, install.wim index 2, |
| answer file on removable media, autologon as Administrator |
| ``` |
| |
| The image-apply phase moved roughly 9 GB to disk in about thirteen minutes, and the |
| desktop was up a few minutes after that. |
| |
| **What was installed inside it.** |
| |
| ``` |
| Python 3.12.8 (embeddable distribution, unpacked to C:\tiny\py) |
| pytest, pillow 13.0.0, mss 10.2.0 |
| numpy 2.5.4, onnxruntime 1.31.1, opencv-python 5.0.0.93, rapidocr-onnxruntime 1.4.4 |
| the harness itself, installed into site-packages |
| ``` |
| |
| **Results.** |
| |
| | Configuration | Result | Wall clock | |
| |---|---|---| |
| | Harness suite, no OCR engine | **118 passed, 3 failed, 4 skipped** | 7.8 s | |
| | Harness suite, OCR engine installed, `test_perception.py` excluded | **109 passed, 3 skipped, exit 0** | 21 min | |
| | `test_perception.py` (OCR inference) | runs; roughly ten minutes per OCR pass | — | |
| |
| None of the three failures in the first row is a harness defect: two are |
| `ModuleNotFoundError: numpy` and the third is the guard bug documented below. With the |
| OCR engine installed, the same suite exits **0**. (Those Windows counts predate `trace.py` and 2.0; the harness suite has since grown to 284 tests.) |
| |
| **The emulated-OCR caveat.** onnxruntime requires AVX2, and QEMU's software emulation of |
| AVX2 is roughly two orders of magnitude slower than native. A suite that finishes in 17 |
| seconds on Linux takes over half an hour on this guest, and the time is spent entirely |
| inside `onnxruntime_inference_collection.py`. That is a property of emulation, not of the |
| harness: on real hardware — including a GitHub `windows-latest` runner — it does not |
| apply. |
| |
| ### What running on Windows caught |
| |
| Four things surfaced by actually running there. Three are genuine deployment requirements |
| that no Linux-only test run can reveal: |
| |
| | Finding | Symptom | Fix | |
| |---|---|---| |
| | **onnxruntime needs AVX2** | `Windows fatal exception: code 0xc000001d` (`STATUS_ILLEGAL_INSTRUCTION`) the moment `onnxruntime` is imported, because QEMU's default `qemu64` CPU advertises no AVX | boot the guest with `-cpu Skylake-Client` (any AVX2 model will do) | |
| | **onnxruntime needs the MSVC runtime** | `ImportError: DLL load failed while importing onnxruntime_pybind11_state` on a clean Server 2022 | install `vc_redist.x64.exe` in the guest | |
| | **`pyautogui` ships as an sdist only** | `pip install pytest pillow mss pyautogui` fails as a whole, so `pytest` never arrives and the suite reports `No module named pytest` | install `pyautogui` in a separate step from the wheel-only packages | |
| | **A guard that never fires** | `test_screenshot_replay_needs_no_display` fails instead of skipping when no OCR engine is present: it guards on `detector_available()[0]`, which is `True` for the grid fallback | guard on the OCR detector specifically | |
| |
| The last entry is a defect in this repository rather than in the environment, and it is |
| exactly the kind of thing that only surfaces on a machine without an OCR engine. |
| |
| ## The test suite, in full |
| |
| `pytest tests` collects **284 tests**. 282 run headless — no display, no network, no |
| weights. The last 2 are the opt-in live-desktop tests, skipped unless |
| `TINY_COMPUTER_AGENT_LIVE_TEST=1`. On a box without PyGObject, two of the 282 *fail* rather |
| than skip: they assert the AT-SPI reader is reachable, and that is the honest outcome. |
| |
| | File | Tests | What it proves | |
| |---|---:|---| |
| | `tests/test_actions.py` | 13 | the closed vocabulary — every action is validated before anything executes | |
| | `tests/test_act.py` | 8 | `DryRunExecutor` is inert; the real executor refuses until explicitly armed | |
| | `tests/test_observe.py` | 8 | element/observation geometry, `FakeScreen`, `ImageScreen`, the capture fallback | |
| | `tests/test_perception.py` | 13 | OCR labels a real image with real coordinates; the grid fallback; `AutoDetector` | |
| | `tests/test_brain.py` | 18 | the numbered `DecisionMenu`, lenient parsing, echo detection, `MockBrain`, `PlannerBrain` | |
| | `tests/test_loop.py` | 14 | `AgentLoop` / `Episode` / `Step`, the no-progress guard, the step budget | |
| | `tests/test_verify.py` | 7 | success is only claimed when a verifier can actually see it | |
| | `tests/test_safety.py` | 11 | deny lists, the kill switch, the blind-grid-click guard | |
| | `tests/test_grid_fallback.py` | 11 | the fallback index announces itself instead of guessing | |
| | `tests/test_desktop.py` | 10 | display detection, the preflight checks, `doctor` | |
| | `tests/test_cli.py` | 16 | the command line: `--doctor`, `--image`, `--live`, and the 2.0 `skills` / `batch` subcommands | |
| | `tests/test_trace.py` | 28 | traces: a run recorded with no pixels, replayed against another brain, and scored against ground truth | |
| | `tests/test_models.py` | 7 | the model backends: lazy loading, device choice, honest refusals | |
| | `tests/test_api_surface.py` | 4 | the public API is a promise: everything advertised is importable, and nothing public is left out | |
| | `tests/test_decoding.py` | 24 | the mask, the answer space, the retry ladder, the vote - and that an echo is refused | |
| | `tests/test_accessibility.py` | 26 | role filtering, the coordinate contract, the fallback chain, and real AT-SPI | |
| | `tests/test_diff.py` | 8 | what changed between two frames: added, removed, moved, and the verifier that remembers | |
| | `tests/test_stepwise.py` | 11 | begin / step / pause / resume, and that it equals the batch loop | |
| | `tests/test_ocr_guard.py` | 4 | the OCR guard really skips with no engine - the guard that never fired | |
| | `tests/test_gates.py` | 8 | the operator's yes: whitelists, ask-once gates, and a refusal that is a step | |
| | `tests/test_skills.py` | 17 | only a verified run becomes a skill; re-grounding by label; aiming at nothing raises | |
| | `tests/test_runner.py` | 10 | a queue, a JSONL resume file, and a report counts what was verified | |
| | `tests/test_live_desktop.py` | 2 | **needs a display, opt-in**: OCR finds a real button at the pixels it was drawn at, and a real click makes the app react | |
| |
| ```bash |
| pytest tests -q # 274 passed, 8 skipped, 2 failed only where |
| # PyGObject/AT-SPI is missing (gir1.2-atspi-2.0) |
| TINY_COMPUTER_AGENT_LIVE_TEST=1 pytest tests -q # + the 2 live-desktop tests, on a desktop |
| ``` |
| |
| ## Install |
| |
| ```bash |
| pip install -e ".[ocr]" # labels from pixels (RapidOCR: no system binary) |
| pip install -e ".[desktop,ocr]" # + real capture and real input |
| pip install -e ".[vision,ocr]" # + the tiny model brains |
| pytest # 284 tests, no display or network needed |
| ``` |
| |
| The accessibility detector needs system packages rather than an extra: |
| |
| ```bash |
| apt install gir1.2-atspi-2.0 at-spi2-core gir1.2-gtk-3.0 |
| pip install PyGObject |
| python -m tiny_computer_agent --doctor --detector best # is the tree reachable? |
| ``` |
| |
| ## Run it on your machine |
| |
| ```bash |
| python -m tiny_computer_agent --doctor # what can this machine do? |
| python -m tiny_computer_agent --image example # replay a screenshot, dry run |
| python -m tiny_computer_agent --live # your real screen, dry run |
| python -m tiny_computer_agent --live --real-input "click 'OK'" |
| python -m tiny_computer_agent --live --real-input --verify-text "Saved" "click 'Save'" |
| ``` |
| |
| `--doctor` is the first thing to run on a new machine: it checks the display session, |
| capture backend, pyautogui, the OCR engine, the model runtime and the kill switch, and |
| prints the fix for anything missing. Real input needs **two** deliberate acts: |
| |
| ```bash |
| export TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1 |
| ``` |
| |
| **Pass `--verify-text`.** Without it, a successful action that removes its own target |
| (clicking Save closes the dialog) is indistinguishable from a failure, and the episode is |
| recorded as one. With it, the harness only reports success when it can see the text it |
| was given. |
| |
| ### No desktop? Use a virtual one |
| |
| Xvfb is enough for everything except a real model's hallucinations: |
| |
| ```bash |
| Xvfb :99 -screen 0 1280x800x24 & |
| export DISPLAY=:99 |
| TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py |
| ``` |
| |
| That test starts its own Tk window, asserts OCR finds the button at the coordinates it |
| was actually drawn at, clicks it for real, and checks the app reacted. It is opt-in |
| because it moves your mouse — run it on a throwaway desktop. |
| |
| ### Testing on Windows - three ways, and which to use |
| |
| The verified Windows run above used method 2. All three remain valid: |
| |
| | Method | Real Windows desktop? | Cost | Status | |
| |---|---|---|---| |
| | **GitHub Actions `windows-latest`** | yes, interactive session | free for public repos, minutes | workflow committed at `.github/workflows/live-desktop.yml` | |
| | **QEMU/TCG VM on a Linux host** | yes, fully emulated | ~25 min to install, hours to run OCR | **used for the results above** | |
| | **A cloud Windows box over RDP** | yes, native | paid by the hour | untested here | |
| |
| Nothing in the harness is Linux-specific: `pyautogui` uses `SendInput` on Windows and |
| `GrabScreen` already falls back to Pillow's `ImageGrab` there. Two things can only fail on |
| a real Windows desktop, which is why these runs exist: **DPI scaling** (that is what |
| `--scale` is for) and the native input backend. At 125% or 150% display scaling, expect the |
| coordinate assertion in `test_live_desktop.py` to be the first thing to fail - that is the |
| signal, not a nuisance. |
| |
| **Reproducing the local VM.** With no KVM, a Windows guest is still perfectly possible - |
| it is simply slow. The working recipe, including the two ingredients that onnxruntime |
| demands: |
| |
| ```bash |
| # 1. install media, plus an answer file on removable media |
| qemu-img create -f qcow2 win.qcow2 64G |
| qemu-system-x86_64 -accel tcg,thread=multi -m 8G -smp 8 -machine pc \ |
| -cpu Skylake-Client \ |
| -drive file=win.qcow2,if=none,id=hd0,format=qcow2,cache=writeback \ |
| -device ide-hd,drive=hd0,bus=ide.0 \ |
| -drive file=win2022.iso,if=none,id=cd0,media=cdrom,readonly=on \ |
| -device ide-cd,drive=cd0,bus=ide.1 \ |
| -drive if=none,id=usb,format=raw,file=unattend.img \ |
| -device usb-ehci,id=ehci -device usb-storage,bus=ehci.0,drive=usb \ |
| -netdev user,id=n0 -device e1000,netdev=n0 \ |
| -vga std -display none -vnc :1 -boot order=d,menu=off |
| |
| # 2. inside the guest, after autologon |
| # vc_redist.x64.exe is required before onnxruntime will load |
| curl.exe -s -o C:\tiny\vc_redist.x64.exe <vc_redist-url> && C:\tiny\vc_redist.x64.exe /install /quiet /norestart |
| ``` |
| |
| Serve files to the guest over QEMU's user-mode networking (the host is `10.0.2.2`), which |
| avoids typing long commands into the emulated console. Two practical notes from doing |
| this: an answer file with non-canonical XML namespaces is **silently ignored** by Setup |
| (you land on the language page), and keyboard input over VNC **drops the shift modifier** |
| (`_` becomes `-`, `:` becomes `;`) - the QEMU monitor's `sendkey` handles shifts correctly. |
| |
| |
| ## How it works |
| |
| | Module | Job | |
| |---|---| |
| | `actions.py` | The closed vocabulary — `click` `type` `key` `scroll` `wait` `done`, validated before anything executes | |
| | `observe.py` | `Element`/`Observation`, `FakeScreen`, `ImageScreen`, `GrabScreen` (mss → Pillow fallback) | |
| | `perception.py` | Detectors: `OcrDetector` (RapidOCR), `GridDetector`, `AutoDetector`, `IndexedScreen` | |
| | `brain.py` | `DecisionMenu` (the numbered menu), `ModelBrain`, lenient `parse_decision`, echo detection, `MockBrain`, `PlannerBrain` | |
| | `models.py` | `VLMBrain` (SmolVLM-256M) and `TextPlannerBrain` (Qwen2.5-0.5B), lazily loaded | |
| | `act.py` | `DryRunExecutor` (the default) and `PyAutoGuiExecutor` | |
| | `safety.py` | `Policy`: deny lists, kill switch, step budget, the blind-grid-click guard | |
| | `loop.py` | `AgentLoop`, `Episode`, `Step`, the no-progress guard and the verifier | |
| | `desktop.py` | Preflight checks, `doctor`, and `build_agent` in one call | |
| | `demo.py` | The synthetic desktop used by tests and `--image example` | |
| | `trace.py` | Portable traces: record a run with no pixels, replay it offline, score the choice against ground truth | |
| | `decoding.py` | Constrained decoding: mask the logits to a menu number or a yes/no, so an echo is impossible | |
| | `accessibility.py` | The accessibility tree: `AtspiReader` and `AtspiDetector`, roles and real extents | |
| | `diff.py` | What changed between two frames: `Delta` (added / removed / moved) and `ChangeVerifier`, so "the dialog closed" is checkable | |
| | `skills.py` | **2.0** - Skills: a *verified* run written down once, replayed by `SkillBrain` with no model. Labels, never coordinates | |
| | `gates.py` | **2.0** - Approval gates: `AllowOnlyLabels`, `InteractiveGate` (yes / no / always / never), `AllOf` / `AnyOf`. A refusal is a step | |
| | `runner.py` | **2.0** - `TaskRunner`: a queue of goals, a JSONL resume file, and a `RunReport` that counts what was *verified* | |
| | `showcase.py` | The capability gallery: one real screenshot per capability, checked against the public API | |
| | `measure_vlm.py` | Measures the four ways to ask a model, on a real screen, and prints the table | |
| | `evaluate_daily.py` | A day's tasks against a real app, verified by the app's own log | |
| |
| ## The marimo console |
| |
| The loop is a library, but it is also a *session* you can drive one turn at a time - |
| which is what a console needs: a live screen preview, an action log, and |
| step / pause / resume. |
| |
| <img src="gallery/console.png" width="900"> |
| |
| `AgentLoop.run()` is the batch form; `begin()`, `step_once()`, `pause()`, `resume()` |
| and `finished` are the stepwise one. `tests/test_stepwise.py` asserts the two cannot |
| drift apart - the same scenario has to produce the same episode either way. The |
| console cell in this notebook wires that to marimo widgets: |
| |
| - **live preview** - the current screen with the element index drawn over it, so you |
| can see what the agent sees and what it thinks is clickable |
| - **action log** - one row per step: the action, and the outcome or the refusal |
| - **step / run / reset** - `mo.ui.run_button`, so an auto-refresh can never fire a |
| step by accident |
| - **pause** - respected *between* steps: the episode stays open, and a single step |
| can still be taken through it |
| - **panic** - creates the same kill-switch file the CLI checks, so the next step |
| refuses to run |
| |
| Switch the screen to "this machine, live" and it is the real desktop, indexed through |
| OCR or through the accessibility tree. |
| |
| ## What the machine already knows: the accessibility tree |
| |
| OCR reads pixels and *guesses* which marks are widgets. An operating system's |
| accessibility tree simply knows: every application that publishes one advertises each |
| button with a role, a name, and its real screen extents. |
| |
| <img src="gallery/atspi-index.png" width="860"> |
| |
| `AtspiDetector` reads that tree through PyGObject's `Atspi` typelib and keeps only the |
| roles worth clicking - a frame and a panel have names too, and offering them to a |
| model is how you get an agent that clicks the window furniture. The coordinates come |
| back as absolute screen pixels, so `absolute_coordinates` tells `IndexedScreen` not to |
| re-map them through a downscale. |
| |
| <img src="gallery/desk-atspi.png" width="860"> |
| |
| That click was found **without reading a single pixel**: the window's own status label |
| reads `last action: Save` afterwards, and the app wrote `Save` to its marker file. |
| `--detector best` tries the tree first, then OCR, then the numbered grid. |
| |
| **Honest limits.** AT-SPI is Linux-only here; **UI Automation is not implemented**, and |
| `accessibility_available("uia")` says exactly that rather than pretending - so an agent |
| on Windows keeps using OCR. Tk barely publishes a tree and a canvas publishes none, |
| which is why this is a detector and not a replacement. |
| |
| ## Can it do a day's work? |
| |
| The honest way to answer "could this replace a person at a PC?" is to give it a |
| person's tasks and check the result against **the application's own record**, not the |
| agent's opinion of itself. Eleven of them, on a real GTK desktop with the real mouse: |
| |
| ``` |
| button yes 0.7 Click Save |
| right-button yes 0.4 Click Cancel, not the button next to it |
| type yes 1.8 Type a new file name and save |
| shortcut yes 0.7 Press Ctrl+S |
| menu yes 0.7 Open the File menu and choose Save As |
| scroll yes 1.5 Scroll the list and pick a row |
| second-window yes 0.7 Open Settings and press Apply there |
| see yes 0.1 Read what the window says |
| verify yes 0.7 Check the save actually happened |
| refuse yes 0.1 Do not press the Terminal button |
| kill-switch yes 0.1 Stop a live run mid-plan |
| |
| 11/11 tasks completed, verified by the application's own log |
| ``` |
| |
| Run it yourself with `python evaluate_daily.py` (`--only menu` for one task). |
| |
| **What that number does and does not mean.** The first run of this suite scored |
| **5/11**, and the failures were not the model. A coordinate bug in the detector |
| *chain* was shifting every accessibility click by the capture's scale factor: |
| `--detector best` wraps the tree detector inside `AutoDetector`, which reported |
| "frame coordinates" for elements that were already screen pixels, so `IndexedScreen` |
| re-mapped them through the downscale and the click landed beside the button. Two more |
| real gaps surfaced with it - an unnamed GTK list row was invisible to the index (so |
| nothing in a list could be clicked at all), and two GTK instances sharing an |
| application id made the second launch silently do nothing. All three are fixed and |
| pinned by tests. |
| |
| So the machine half works: it can see, decide, act and verify a day's operations, and |
| it refuses the dangerous ones. What this suite deliberately does **not** measure is |
| *choosing* the actions - nine of the eleven tasks were driven by a scripted action |
| list, because that is what isolates the harness from the brain. Choosing is measured |
| in [asking a tiny model in four ways](#asking-a-tiny-model-in-four-ways-measured): |
| 4/4 goals right for a 0.5B text planner over the accessibility index in `rank` mode, |
| and 1-2/4 for a 256M VLM. |
| |
| Caveats, stated plainly: this is a Linux GTK desktop under Xvfb with **no window |
| manager** (so the keyboard tasks had to move the pointer into the window first, |
| because X without a manager focuses by pointer); Windows was not exercised; the tasks |
| are single-application with no overlapping windows, no animations and no network; and |
| eleven tasks are a sample, not a survey. |
| |
| ## Capability gallery |
| |
| **74 capabilities, one screenshot each - and 173 of the package's 180 pre-2.0 public callables (functions, methods, classes and protocols) appear in at least one of them**, together with 10 of its public constants. The 2.0 additions are not pictured yet: **84 new public callables** (skills, approval gates, the batch runner and the structural diff) plus four new CLI entry points. The counts are read off `gallery/index.json`, which records what each picture shows. |
| |
| Nothing here is a mock-up. Each picture is produced by [`showcase.py`](showcase.py): a command or a call is run for real, and its actual output - or the real desktop it acted on - is what you see. The desktop shots are real screen captures on a real X server (Xvfb, 1280x800), including a real mouse click verified by the target application's own reaction. |
| |
| > **An honest note on the platform.** These were captured on Linux, because the sandbox that builds this card has no Windows: no KVM, no QEMU and no Windows image. The harness is OS-agnostic (pyautogui uses `SendInput` on Windows, and `GrabScreen` falls back to Pillow's `ImageGrab`), so these are the same code paths - but they are not Windows pixels. There are no Windows pictures here, and that workflow does not by itself produce any: its `windows-latest` job does build the same gallery on real Windows pixels, but it uploads them as a CI **artifact** (`gallery-windows`) - nothing downloads it, re-prefixes the files `win-*`, or appends a section to this card, so an artifact is as far as they get. The Windows screenshots from the guest described [below](#verified-on-real-windows) were lost with the build container that produced them. |
| |
| Regenerate it with `python showcase.py` (`--list` to see the ids, `--only <id>` for a subset, `--card` to rewrite this section). It runs the calls for real, so a broken capability fails loudly instead of shipping a stale picture. |
| |
| | area | capabilities | |
| |---|---:| |
| | Action vocabulary | 3 | |
| | Decisions | 7 | |
| | Execution | 2 | |
| | The loop | 7 | |
| | Safety | 4 | |
| | Screens | 6 | |
| | Perception | 6 | |
| | Preflight | 4 | |
| | Models | 5 | |
| | Traces & scoring | 7 | |
| | Command line | 9 | |
| | Real desktop | 7 | |
| | Extensibility | 1 | |
| |
| --- |
| |
| ### Action vocabulary |
| |
| **The six verbs, and only six** |
| |
| *Every action is validated on construction, before anything touches the machine.* |
| |
| <img src="gallery/vocab-verbs.png" width="860"> |
| |
| `shows:` `ACTION_KINDS`, `ACTION_MENU`, `Action`, `Action.describe`, `Action.to_dict`, `parse_action` |
| |
| **Malformed actions are refused, not guessed at** |
| |
| *An unknown verb or a missing target raises before the machine is touched.* |
| |
| <img src="gallery/vocab-refusals.png" width="860"> |
| |
| `shows:` `ActionError`, `Action`, `parse_action` |
| |
| **The contract handed to a model** |
| |
| *The prompt-side menu: one JSON object, six shapes, no invented verbs.* |
| |
| <img src="gallery/vocab-menu.png" width="860"> |
| |
| `shows:` `ACTION_MENU` |
| |
| ### Decisions |
| |
| **A short numbered menu, in reading order** |
| |
| *A 256M model cannot emit coordinates; it can pick a number from twelve.* |
| |
| <img src="gallery/menu-numbered.png" width="860"> |
| |
| `shows:` `DecisionMenu`, `DecisionMenu.render`, `MenuItem`, `MenuItem.label` |
| |
| **Menu lookup tolerates what models actually send** |
| |
| *1, "2", 2.0 and "Save." all resolve; a stray word resolves to nothing.* |
| |
| <img src="gallery/menu-lookup.png" width="860"> |
| |
| `shows:` `DecisionMenu.item`, `DecisionMenu.find`, `DecisionMenu.mentioned_in` |
| |
| **Reading a decision out of a small model's reply** |
| |
| *The LAST JSON object wins, because a model that echoes its prompt answers last.* |
| |
| <img src="gallery/decision-json.png" width="860"> |
| |
| `shows:` `parse_decision`, `extract_json_object`, `extract_json_objects`, `DecisionMenu` |
| |
| **An echoed prompt is refused; a right answer in the wrong field is recovered** |
| |
| *Real 256M/0.5B failure modes, both caught: no blind click either way.* |
| |
| <img src="gallery/decision-echo-guard.png" width="860"> |
| |
| `shows:` `parse_decision`, `BrainError`, `DecisionMenu.mentioned_in` |
| |
| **A garbled verb with a valid label is still usable** |
| |
| *A 256M model really does emit {"kind": "file", "label": "Save"}.* |
| |
| <img src="gallery/decision-garbled.png" width="860"> |
| |
| `shows:` `parse_decision`, `BrainError` |
| |
| **MockBrain: replay a fixed script, exactly once** |
| |
| *Deterministic and offline - the workhorse for tests, demos and replays.* |
| |
| <img src="gallery/brain-mock.png" width="860"> |
| |
| `shows:` `MockBrain`, `MockBrain.decide`, `MockBrain.remaining` |
| |
| **PlannerBrain: no weights, no download, and it gives up honestly** |
| |
| *The proof the harness works end to end, and a real fallback when no model is wanted.* |
| |
| <img src="gallery/brain-planner.png" width="860"> |
| |
| `shows:` `PlannerBrain`, `PlannerBrain.decide`, `GoalUnreachable` |
| |
| ### Execution |
| |
| **Dry run resolves every click to pixels - and touches nothing** |
| |
| *The default everywhere: read a whole episode before anything clicks.* |
| |
| <img src="gallery/exec-dry-run.png" width="860"> |
| |
| `shows:` `DryRunExecutor`, `DryRunExecutor.execute`, `resolve_point`, `ActionResult` |
| |
| **Real input refuses to guess a pixel for a label it cannot find** |
| |
| *This is the gate between a hallucinated label and 'Delete account'.* |
| |
| <img src="gallery/exec-refuses-unresolved.png" width="860"> |
| |
| `shows:` `ActionResult.summary`, `InputUnavailable`, `PyAutoGuiExecutor`, `PyAutoGuiExecutor.execute` |
| |
| ### The loop |
| |
| **observe -> think -> act -> verify, traced end to end** |
| |
| *One call runs a whole episode; every step is kept for audit.* |
| |
| <img src="gallery/loop-end-to-end.png" width="860"> |
| |
| `shows:` `AgentLoop`, `AgentLoop.run`, `Episode`, `Episode.render`, `Episode.to_rows`, `Step`, `Step.line`, `Step.ok` |
| |
| **A hard step budget, so a run cannot wander** |
| |
| *The budget ends the episode and says so; nothing executes past it.* |
| |
| <img src="gallery/loop-budget.png" width="860"> |
| |
| `shows:` `Policy`, `Policy.step_budget`, `AgentLoop.run` |
| |
| **Repeating one action against an unchanged screen is stopped** |
| |
| *A stuck agent is not a persistent one - it is stopped and told why.* |
| |
| <img src="gallery/loop-no-progress-guard.png" width="860"> |
| |
| `shows:` `AgentLoop.run`, `Step`, `Step.error` |
| |
| **A kill-switch file aborts before the next action** |
| |
| *Checked before every single action: create the file, the run stops.* |
| |
| <img src="gallery/loop-kill-switch.png" width="860"> |
| |
| `shows:` `Policy.kill_switch_path`, `Policy.should_stop`, `AgentLoop.run` |
| |
| **A dead screen or a broken brain is an episode, not a traceback** |
| |
| *Every failure path is recorded as a failed step with a reason.* |
| |
| <img src="gallery/loop-survives-failures.png" width="860"> |
| |
| `shows:` `AgentLoop.run`, `Episode.stop_reason`, `GoalUnreachable`, `BrainError` |
| |
| **Success is evidence, not the model's word** |
| |
| *Without a verifier, a successful action that removes its own target looks like a failure.* |
| |
| <img src="gallery/loop-verifier.png" width="860"> |
| |
| `shows:` `AgentLoop.run`, `Episode.success`, `Observation.text` |
| |
| **Every step can be streamed out as it happens** |
| |
| *This is the hook a live console, a progress bar or a websocket uses.* |
| |
| <img src="gallery/loop-on-step.png" width="860"> |
| |
| `shows:` `AgentLoop.run`, `Step.line`, `on_step` |
| |
| ### Safety |
| |
| **Typing from the deny list never reaches the keyboard** |
| |
| *Checked before execution; a refusal ends the episode and clicks nothing.* |
| |
| <img src="gallery/safety-deny-text.png" width="860"> |
| |
| `shows:` `Policy`, `Policy.check`, `PolicyViolation`, `DEFAULT_DENY_TEXT` |
| |
| **Clicking a denied widget is refused, however the label is spelled** |
| |
| *Terminal, keychain, wallet, password manager, bank - matched on the label.* |
| |
| <img src="gallery/safety-deny-label.png" width="860"> |
| |
| `shows:` `Policy.deny_labels`, `Policy.check`, `Observation.find` |
| |
| **Key chords on the deny list are never pressed** |
| |
| *ctrl+alt+t, ctrl+alt+delete and ctrl+shift+escape are refused by default.* |
| |
| <img src="gallery/safety-deny-keys.png" width="860"> |
| |
| `shows:` `Policy.deny_keys`, `Policy.check`, `PolicyViolation` |
| |
| **A numbered grid cell is a blind click: allowed to plan, refused for real** |
| |
| *With real input the blind guess is refused unless you pass --allow-grid.* |
| |
| <img src="gallery/safety-blind-grid.png" width="860"> |
| |
| `shows:` `Policy.allow_grid_clicks`, `Policy.check`, `GridDetector` |
| |
| ### Screens |
| |
| **A scripted screen, for testing the loop with no display** |
| |
| *The final frame repeats forever, so a runaway agent cannot exhaust the script.* |
| |
| <img src="gallery/screen-fake.png" width="860"> |
| |
| `shows:` `FakeScreen`, `FakeScreen.capture`, `Observation` |
| |
| **A still image (or a sequence) replayed as a screen** |
| |
| *Lets the whole pipeline be exercised with no display, and a failure be replayed.* |
| |
| <img src="gallery/screen-image.png" width="860"> |
| |
| `shows:` `ImageScreen`, `ImageScreen.capture`, `fake_desktop` |
| |
| **Real capture: monitors, downscaling, and HiDPI mapping** |
| |
| *Frame pixel -> screen pixel is preserved, so a click cannot drift.* |
| |
| <img src="gallery/capture-options.png" width="860"> |
| |
| `shows:` `GrabScreen`, `GrabScreen.capture`, `GrabScreen.map_to_screen`, `ScreenUnavailable` |
| |
| **Labels are compared on their alphanumeric core** |
| |
| *OCR loses spaces, models add punctuation - so nothing matches by exact string.* |
| |
| <img src="gallery/element-matching.png" width="860"> |
| |
| `shows:` `Element`, `Element.center`, `Element.matches`, `Observation.find`, `normalise_label` |
| |
| **The compact text a model is shown** |
| |
| *This is the whole world as the brain sees it - capped, and honest when empty.* |
| |
| <img src="gallery/observation-summary.png" width="860"> |
| |
| `shows:` `Observation.summary` |
| |
| **The built-in synthetic desktop** |
| |
| *Draws a screenshot-like desktop offline - the same image --image example replays.* |
| |
| <img src="gallery/fake-desktop.png" width="860"> |
| |
| `shows:` `fake_desktop`, `write_fake_desktop`, `OcrDetector` |
| |
| ### Perception |
| |
| **Probing for a detector, and building one by name** |
| |
| *The doctor and the CLI both ask this before promising anything.* |
| |
| <img src="gallery/detector-available.png" width="860"> |
| |
| `shows:` `detector_available`, `make_detector`, `DetectorUnavailable`, `Detector` |
| |
| **OCR turns a screenshot into labelled, clickable elements** |
| |
| *The model never has to guess where anything is: this is the grounding.* |
| |
| <img src="gallery/ocr-labels.png" width="860"> |
| |
| `shows:` `OcrDetector`, `OcrDetector.detect`, `Element` |
| |
| **Three knobs keep dialog prose out of the menu** |
| |
| *Every extra line of noise is one more chance for a small model to click prose.* |
| |
| <img src="gallery/ocr-filters.png" width="860"> |
| |
| `shows:` `OcrDetector.min_score`, `OcrDetector.max_label_chars`, `OcrDetector.min_label_chars` |
| |
| **A numbered grid, for screens with no text at all** |
| |
| *An agent with an empty index has no legal moves; a grid always has moves.* |
| |
| <img src="gallery/grid-detector.png" width="860"> |
| |
| `shows:` `GridDetector`, `GridDetector.detect` |
| |
| **OCR when there is text, a grid when there is none** |
| |
| *The default: useful with nothing installed, better the moment OCR appears.* |
| |
| <img src="gallery/auto-detector.png" width="860"> |
| |
| `shows:` `AutoDetector`, `AutoDetector.detect` |
| |
| **IndexedScreen: capture + detect, composed** |
| |
| *One capture() call gives the loop an index it can act on.* |
| |
| <img src="gallery/indexed-screen.png" width="860"> |
| |
| `shows:` `IndexedScreen`, `IndexedScreen.capture`, `Observation.summary` |
| |
| ### Preflight |
| |
| **Is there a desktop session to drive at all?** |
| |
| *The first question the CLI and the doctor both ask.* |
| |
| <img src="gallery/display-available.png" width="860"> |
| |
| `shows:` `display_available` |
| |
| **Every precondition, one list, each with the fix** |
| |
| *Eight checks; a required one that fails is a blocking problem.* |
| |
| <img src="gallery/run-checks.png" width="860"> |
| |
| `shows:` `run_checks`, `Check`, `Check.render` |
| |
| **python -m tiny_computer_agent --doctor** |
| |
| *The first thing to run on a new machine. Exits non-zero when not ready.* |
| |
| <img src="gallery/cli-doctor.png" width="860"> |
| |
| `shows:` `detector_available`, `display_available`, `doctor`, `report` |
| |
| **One call wires screen + detector + brain + executor + policy** |
| |
| *Defaults are the safe ones: dry run, a step budget, and a kill-switch file.* |
| |
| <img src="gallery/build-agent.png" width="860"> |
| |
| `shows:` `build_agent`, `Policy`, `AgentLoop` |
| |
| ### Models |
| |
| **Which model runtime is present, and are the weights cached?** |
| |
| *Weights are loaded lazily, so asking is free.* |
| |
| <img src="gallery/models-deps.png" width="860"> |
| |
| `shows:` `model_dependencies`, `model_cached`, `MODEL_BRAINS`, `make_model_brain`, `DEFAULT_TEXT_MODEL`, `DEFAULT_VISION_MODEL`, `BrainError` |
| |
| **Device and dtype selection** |
| |
| *CUDA if it is there, CPU otherwise; the harness never needs a GPU.* |
| |
| <img src="gallery/models-device.png" width="860"> |
| |
| `shows:` `resolve_device`, `pick_dtype` |
| |
| **The prompt a model actually receives - and how it learns from a dead end** |
| |
| *Placeholders, not worked examples: a weak model echoes the prompt, and an echo fails validation.* |
| |
| <img src="gallery/models-prompt.png" width="860"> |
| |
| `shows:` `ModelBrain`, `ModelBrain.build_prompt`, `ModelBrain.decide` |
| |
| **A second decision is told what already failed** |
| |
| *'you already tried this and the screen did NOT change' is the most useful line a small model gets.* |
| |
| <img src="gallery/models-prompt-history.png" width="860"> |
| |
| `shows:` `ModelBrain`, `ModelBrain.build_prompt`, `ModelBrain.history` |
| |
| **The two real model backends, and lazy loading** |
| |
| *Constructing a brain costs nothing; the weights arrive on the first decide().* |
| |
| <img src="gallery/models-swap.png" width="860"> |
| |
| `shows:` `TextPlannerBrain`, `VLMBrain`, `TextPlannerBrain.unload`, `VLMBrain.unload`, `ModelBrain.loaded`, `make_model_brain` |
| |
| ### Traces & scoring |
| |
| **A task with one right answer, and the frames to show** |
| |
| *Ground truth lives with the task, so a run can be scored at all.* |
| |
| <img src="gallery/scenario.png" width="860"> |
| |
| `shows:` `Scenario`, `Scenario.label`, `Scenario.screen`, `Trace.render`, `run_scenario` |
| |
| **An episode becomes a portable, pixel-free trace** |
| |
| *JSON, so it diffs in review and replays on a machine with nothing installed.* |
| |
| <img src="gallery/trace-record.png" width="860"> |
| |
| `shows:` `Trace`, `Trace.blocked`, `Trace.choice`, `Trace.decided`, `Trace.decision_step`, `Trace.frames`, `Trace.load`, `Trace.name`, `Trace.record`, `Trace.save`, `Trace.to_dict`, `Trace.to_json`, `TraceStep`, `TraceStep.observation`, `TraceStep.to_dict` |
| |
| **Replay a recorded trace against a different brain** |
| |
| *The frames are the fixture; the brain is the only variable. Nothing is captured or clicked.* |
| |
| <img src="gallery/trace-replay.png" width="860"> |
| |
| `shows:` `Trace.replay`, `Trace.observations`, `score_trace` |
| |
| **Counting how often the choice was right** |
| |
| *Scored on normalised text, so OCR spacing is forgiven - the same bargain the index makes.* |
| |
| <img src="gallery/score-trace.png" width="860"> |
| |
| `shows:` `score_trace`, `score`, `score_replay`, `Verdict`, `Verdict.line` |
| |
| **Verdicts collected into an accuracy** |
| |
| *The number the whole module exists for: which brain is right, and how often.* |
| |
| <img src="gallery/scoreboard.png" width="860"> |
| |
| `shows:` `Scoreboard`, `Scoreboard.accuracy`, `Scoreboard.accuracy_for`, `Scoreboard.add`, `Scoreboard.blocked`, `Scoreboard.brains`, `Scoreboard.by_brain`, `Scoreboard.confusion`, `Scoreboard.earned`, `Scoreboard.render`, `Scoreboard.to_rows`, `Scoreboard.total` |
| |
| **Every scenario through every brain, in one table** |
| |
| *A fresh brain per task, so state like 'already clicked this' cannot leak between them.* |
| |
| <img src="gallery/benchmark.png" width="860"> |
| |
| `shows:` `benchmark`, `Scoreboard.to_rows` |
| |
| **Replay a whole recorded suite against one brain** |
| |
| *The literal roadmap step: the frames are fixed, nothing new is captured.* |
| |
| <img src="gallery/benchmark-traces.png" width="860"> |
| |
| `shows:` `benchmark_traces`, `load_traces`, `Trace.save` |
| |
| ### Command line |
| |
| **--image example (replay a screenshot, dry run)** |
| |
| *No display needed: the whole pipeline over a still image.* |
| |
| <img src="gallery/cli-image.png" width="860"> |
| |
| `shows:` `main`, `build_parser`, `ImageScreen`, `detector_available` |
| |
| **--script (replay a fixed action list instead of a brain)** |
| |
| *The script is validated exactly like a model's answer: nothing skips the gates.* |
| |
| <img src="gallery/cli-script.png" width="860"> |
| |
| `shows:` `MockBrain`, `demo_screen`, `parse_action` |
| |
| **--verify-text (the harness decides success, not the brain)** |
| |
| *Without it, an action that removes its own target looks like a failure.* |
| |
| <img src="gallery/cli-verify-text.png" width="860"> |
| |
| `shows:` `AgentLoop.run`, `Episode.success` |
| |
| **--trace / --expected (record the run, then score it)** |
| |
| *Writes JSON next to the code, and prints the score straight away.* |
| |
| <img src="gallery/cli-trace.png" width="860"> |
| |
| `shows:` `Trace.record`, `Trace.save`, `score_trace` |
| |
| **--no-ocr (index a numbered grid instead)** |
| |
| *Useful on a machine with no OCR engine, or for a canvas app.* |
| |
| <img src="gallery/cli-detector.png" width="860"> |
| |
| `shows:` `GridDetector`, `make_detector`, `detector_available` |
| |
| **Exit codes: 0 success, 1 failure, 2 bad usage, 3 no screen** |
| |
| *A shell can tell 'it worked' from 'it gave up' without parsing any text.* |
| |
| <img src="gallery/cli-exit-codes.png" width="860"> |
| |
| `shows:` `main` |
| |
| **The two deliberate acts before the mouse can move** |
| |
| *--real-input without the env var, and --brain vlm with no pixels: both refuse.* |
| |
| <img src="gallery/cli-refusals.png" width="860"> |
| |
| `shows:` `main`, `ALLOW_REAL_INPUT_ENV` |
| |
| **--live with no display is refused, with the fix** |
| |
| *It tells you what to do instead of dying inside an episode.* |
| |
| <img src="gallery/cli-live-headless.png" width="860"> |
| |
| `shows:` `display_available`, `main` |
| |
| **--live (your real screen, dry run)** |
| |
| *Real capture, real OCR, a real element index - and not one click.* |
| |
| <img src="gallery/cli-live-dry-run.png" width="860"> |
| |
| `shows:` `GrabScreen`, `IndexedScreen`, `AutoDetector`, `main` |
| |
| ### Real desktop |
| |
| **GrabScreen: capture a real desktop** |
| |
| *1280x800 of a real X server. mss first, Pillow's ImageGrab as the fallback.* |
| |
| <img src="gallery/desk-capture.png" width="860"> |
| |
| `shows:` `GrabScreen`, `GrabScreen.capture`, `ScreenUnavailable` |
| |
| **The element index, drawn on the pixels it came from** |
| |
| *Every box is a real OCR word with a real screen coordinate: the model picks a label, not a pixel.* |
| |
| <img src="gallery/desk-ocr-index.png" width="860"> |
| |
| `shows:` `OcrDetector`, `OcrDetector.detect`, `IndexedScreen`, `Element.center` |
| |
| **A real click, and the window's own reaction** |
| |
| *Pixels in, a label chosen, a pixel resolved, a button pressed - and the dialog closed itself.* |
| |
| <img src="gallery/desk-click.png" width="860"> |
| |
| `shows:` `PyAutoGuiExecutor`, `AgentLoop.run`, `Policy.dry_run`, `resolve_point` |
| |
| **Success only when the harness can see it** |
| |
| *The app changed its own title and wrote a marker file; the verifier saw the change.* |
| |
| <img src="gallery/desk-verify.png" width="860"> |
| |
| `shows:` `AgentLoop.run`, `Episode.success`, `Episode.stop_reason` |
| |
| **A screen with no text still has legal moves** |
| |
| *OCR finds nothing on a canvas, so the index degrades to numbered cells - and says so.* |
| |
| <img src="gallery/desk-grid-fallback.png" width="860"> |
| |
| `shows:` `AutoDetector`, `GridDetector`, `OcrDetector` |
| |
| **The safety gate, on a real button, in a real window** |
| |
| *The Terminal button is visible and labelled - and the policy refuses to press it.* |
| |
| <img src="gallery/desk-policy-refusal.png" width="860"> |
| |
| `shows:` `Policy.check`, `Policy.deny_labels`, `PolicyViolation`, `Observation.find` |
| |
| **The same click, driven end to end from the CLI** |
| |
| *--live --real-input --verify-text Saved: capture, OCR, click, verify, exit code.* |
| |
| <img src="gallery/desk-cli-real-input.png" width="860"> |
| |
| `shows:` `main`, `PyAutoGuiExecutor`, `AgentLoop.run` |
| |
| ### Extensibility |
| |
| **Every stage is a Protocol: bring your own screen, brain, detector, executor** |
| |
| *The loop depends on four one-method contracts, not on the shipped implementations.* |
| |
| <img src="gallery/protocols.png" width="860"> |
| |
| `shows:` `Brain`, `Brain.decide`, `ScreenSource`, `ScreenSource.capture`, `Detector`, `Detector.detect`, `Executor`, `Executor.execute`, `ActionResult` |
| |
| ## Why a tiny model can do this at all |
| |
| | Job | Who does it | |
| |---|---| |
| | Finding *where* a button is | OCR / accessibility tree — **not** the model | |
| | Choosing *what* to do | the model, but only by picking a number from a short menu | |
| | Knowing whether it worked | the verifier and the no-progress guard, not the model's word | |
| | Recovering from a bad step | the retry policy in `loop.py`, not the weights | |
| |
| A 256M model cannot emit `(295, 284)` reliably, but it can pick "4" out of five options — |
| and the harness owns the coordinate lookup, the vocabulary check and the definition of |
| success. |
| |
| ## Safety, because this thing has your mouse |
| |
| - Run it in a **VM, container, or throwaway desktop account**, not your real session. |
| - **Dry run is the default**, so you can read a whole episode before anything clicks. |
| - Deny lists for typing (`sudo`, `rm -rf`, "password"), widgets (terminal, keychain, |
| wallet, bank) and key chords. |
| - A **kill-switch file** checked before every action, plus a hard step budget. |
| - **Blind grid clicks are refused with real input** unless you pass `--allow-grid`: when |
| OCR finds nothing, the fallback index is coordinates with no meaning, and a model |
| choosing one is guessing about your desktop. |
| - **A brain's "done" is a claim, not evidence**: pass `--verify-text` and the harness |
| decides success itself. |
| - **Local-first, for real:** pixels and keystrokes never leave the machine. |
| |
| ## What works, and what does not |
| |
| **Works:** real capture, a real OCR index with coordinates that land on the right pixel, |
| a real click an app reacts to, verified success, the closed vocabulary, the policy gates, |
| the no-progress guard, echo detection, the doctor, the CLI, portable traces that score a brain's choices offline, skills learned from verified runs, |
| a resumable task runner, operator approval gates, 284 tests, a 256M VLM that |
| reads a button label from pixels on a clean screenshot — and the same suite passing inside |
| a real Windows Server 2022 guest with no KVM (109 passed, exit 0, with the OCR-inference |
| tests excluded as impractically slow under emulation). |
| |
| **Does not work yet:** reliability on a *live* screen. The 256M VLM echoed the prompt |
| there; the 0.5B text planner picks wrong actions; the accessibility-tree detector is not |
| implemented, so labels come from OCR only; and the harness still trusts a model's `done` |
| when no verifier is given. |
| |
| ## Roadmap |
| |
| - [x] Capture, OCR index, closed vocabulary, dry-run default, policy gates, doctor |
| - [x] A real click on a real window, with a verified outcome |
| - [x] Blind-grid-click guard, echo detection, no-progress guard |
| - [x] The harness suite running on real Windows (Server 2022 guest, no KVM — see above) |
| - [x] Score episodes: replay a trace and count how often the choice was right - see [scoring a brain, offline](#scoring-a-brain-offline) |
| - [x] A capability gallery: one real screenshot per capability, generated by `showcase.py` |
| - [x] The OCR guard skips instead of failing: one tested guard, `conftest.require_ocr` |
| - [x] Stop the 256M VLM echoing the prompt - masked decoding, measured in |
| [asking a tiny model in four ways](#asking-a-tiny-model-in-four-ways-measured) |
| - [x] The marimo console: live screen preview, action log, pause / resume / step |
| - [x] An accessibility-tree detector on Linux (AT-SPI), driving a real click with no OCR |
| - [x] Make a tiny model choose correctly: a 0.5B text planner over the accessibility index in |
| `rank` mode gets **4/4** goals right in 0.9s. A 256M **vision** model does not (1-2/4) - |
| it is below the threshold, which is why the rule-free planner is still the default |
| - [x] A daily-usage evaluation: 11 tasks (menu, scroll, typing, shortcut, second window, |
| verify, refuse, kill switch) against a real GTK app, checked against the application's |
| own log - see [can it do a day's work](#can-it-do-a-days-work) |
| - [x] **2.0** Skills: a verified run is written down once and replayed with no model at all |
| - [x] **2.0** Operator approval gates - a label whitelist, or ask before every action |
| - [x] **2.0** A resumable task queue with a report, and structural verification of "it worked" |
| - [x] **2.0** `skills` and `batch` subcommands, and an example queue at `tasks/day.json` |
| - [ ] A gallery run that pictures the 2.0 additions: the 84 new public callables have no screenshot yet |
| - [ ] **Verify UI Automation** (Windows) against a real Windows desktop. The reader and the |
| detector are implemented and unit-tested against a synthetic control tree, and |
| `accessibility_available("uia")` reports the truth on any machine - but no Windows |
| has run the real backend yet, which is the same GitHub-runner blocker as below. |
| - [ ] Windows run of the *live desktop* test: the workflow is committed, but it needs a |
| GitHub mirror with Actions enabled — Hugging Face does not run GitHub Actions |
| - [ ] A **Windows** run of the gallery: the workflow builds it on `windows-latest` (job |
| `gallery-windows`), with the same GitHub-mirror caveat - and when it runs, its |
| artifact still has to be imported into `gallery/` as `win-*` and spliced in here; |
| no step does that yet |
| |
| ## License |
| |
| MIT — see [LICENSE](LICENSE). |
| ''' |
|
|
| mo.md(readme_md) |
| return (readme_md,) |
|
|
|
|
| @app.cell(hide_code=True) |
| def repo_files(): |
| pyproject_toml = ''' |
| [build-system] |
| requires = ["setuptools>=68"] |
| build-backend = "setuptools.build_meta" |
| |
| [project] |
| name = "tiny-computer-agent" |
| version = "2.0.0" |
| description = "An agentic harness that drives a real computer with a tiny, fully local model - no cloud API. It learns from verified runs." |
| readme = "README.md" |
| requires-python = ">=3.10" |
| license = { text = "MIT" } |
| authors = [{ name = "hkblockchaintechnology" }] |
| keywords = ["computer-use", "gui-agent", "agentic-harness", "local-first", "tiny-models"] |
| classifiers = [ |
| "Development Status :: 3 - Alpha", |
| "Intended Audience :: Developers", |
| "Programming Language :: Python :: 3 :: Only", |
| "Topic :: Scientific/Engineering :: Artificial Intelligence", |
| ] |
| dependencies = [] # the harness core imports nothing outside the standard library |
| |
| [project.optional-dependencies] |
| ocr = ["rapidocr-onnxruntime>=1.3"] # labels from pixels, no system binary |
| desktop = ["mss>=9.0", "pyautogui>=0.9", "pillow>=10"] # capture + real input |
| vision = ["transformers>=4.48", "torch>=2.2", "pillow>=10"] # tiny model brains |
| dev = ["pytest>=8"] |
| |
| [project.scripts] |
| tiny-computer-agent = "tiny_computer_agent.__main__:main" |
| |
| [project.urls] |
| Homepage = "https://huggingface.co/hkblockchaintechnology/tiny-computer-agent" |
| |
| [tool.setuptools] |
| packages = ["tiny_computer_agent"] |
| |
| [tool.pytest.ini_options] |
| testpaths = ["tests"] |
| ''' |
|
|
| license_txt = '''MIT License |
| |
| Copyright (c) 2026 hkblockchaintechnology |
| |
| Permission is hereby granted, free of charge, to any person obtaining a copy |
| of this software and associated documentation files (the "Software"), to deal |
| in the Software without restriction, including without limitation the rights |
| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| copies of the Software, and to permit persons to whom the Software is |
| furnished to do so, subject to the following conditions: |
| |
| The above copyright notice and this permission notice shall be included in all |
| copies or substantial portions of the Software. |
| |
| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| SOFTWARE. |
| ''' |
|
|
| gitignore_txt = '''__pycache__/ |
| *.py[cod] |
| .venv/ |
| venv/ |
| .env |
| *.gguf |
| models/ |
| runs/ |
| traces/ |
| .ipynb_checkpoints/ |
| .DS_Store |
| ''' |
|
|
| mo.md(f"packaging: {len(pyproject_toml) + len(license_txt) + len(gitignore_txt)} chars, " |
| f"v2.0.0 with ocr/desktop/vision extras") |
| return gitignore_txt, license_txt, pyproject_toml |
|
|
|
|
| @app.cell |
| def publish(): |
| REPO_ID = "hkblockchaintechnology/tiny-computer-agent" |
|
|
| publish_button = mo.ui.button( |
| label="⬆️ Publish README.md to the Hub", |
| value=0, |
| on_click=lambda value: value + 1, |
| ) |
|
|
| publish_button |
| return REPO_ID, publish_button |
|
|
|
|
| @app.cell |
| def publish_status( |
| REPO_ID, |
| gitignore_txt, |
| license_txt, |
| publish_button, |
| pyproject_toml, |
| readme_md, |
| ): |
| _status = mo.md( |
| "This notebook owns the project: the card, the packaging files, the package and " |
| "the tests. Click **Publish** to push all of it to the Hub." |
| ).callout(kind="info") |
|
|
| if publish_button.value: |
| _token = os.environ.get("HF_TOKEN") |
| if not _token: |
| _status = mo.md( |
| "No `HF_TOKEN` in the environment. Run `export HF_TOKEN=...` " |
| "(or `hf auth login`), then restart the kernel." |
| ).callout(kind="warn") |
| else: |
| _api = HfApi(token=_token) |
| _root = Path(__file__).parent |
| _notebook = Path(__file__) |
| _files = { |
| "README.md": readme_md, |
| "pyproject.toml": pyproject_toml, |
| "LICENSE": license_txt, |
| ".gitignore": gitignore_txt, |
| "tiny_computer_agent_app.py": _notebook.read_text(encoding="utf-8"), |
| } |
| |
| for _tool in ("showcase.py", "showcase_core.py", "showcase_catalog.py", |
| "measure_vlm.py", "evaluate_daily.py"): |
| if (_root / _tool).is_file(): |
| _files[_tool] = (_root / _tool).read_text(encoding="utf-8") |
| _folders = [f for f in ("tiny_computer_agent", "tests", "gallery", |
| "tasks", "showcase_scenes", "preview") |
| if (_root / f).is_dir()] |
| if (_root / ".github").is_dir(): |
| _folders.append(".github") |
| _missing = [f for f in ("tiny_computer_agent", "tests") if not (_root / f).is_dir()] |
| if _missing: |
| _status = mo.md(f"Nothing to publish: missing {_missing} next to `{_root}`").callout(kind="danger") |
| else: |
| try: |
| for _name, _text in _files.items(): |
| _api.upload_file( |
| path_or_fileobj=_text.encode("utf-8"), |
| path_in_repo=_name, |
| repo_id=REPO_ID, |
| repo_type="model", |
| commit_message=f"Publish {_name} from marimo", |
| ) |
| for _folder in _folders: |
| _api.upload_folder( |
| folder_path=str(_root / _folder), |
| path_in_repo=_folder, |
| repo_id=REPO_ID, |
| repo_type="model", |
| commit_message=f"Publish {_folder}/ from marimo", |
| ignore_patterns=["__pycache__/*", "*.pyc"], |
| ) |
| _status = mo.md( |
| f"✅ Published {len(_files)} files and {len(_folders)} folders to " |
| f"[{REPO_ID}](https://huggingface.co/{REPO_ID})" |
| ).callout(kind="success") |
| except Exception as _e: |
| _status = mo.md(f"Publish failed: `{type(_e).__name__}: {_e}`").callout(kind="danger") |
|
|
| _status |
| return |
|
|
|
|
| @app.cell |
| def card_preview(readme_md): |
| |
| import json |
|
|
| from PIL import Image |
|
|
| _card_root = Path(__file__).parent |
| _gallery_index = json.loads((_card_root / "gallery" / "index.json").read_text()) |
| _preview_shots = sorted((_card_root / "preview").glob("0*.png")) |
|
|
| card_status = mo.md( |
| f"**The published card.** {len(readme_md):,} characters, " |
| f"{readme_md.count('<img src="gallery/')} screenshots covering " |
| f"{len(_gallery_index)} capabilities - every public callable pictured. " |
| f"Live at [hkblockchaintechnology/tiny-computer-agent]" |
| f"(https://huggingface.co/hkblockchaintechnology/tiny-computer-agent)." |
| ) |
|
|
| mo.vstack( |
| [card_status] |
| + ( |
| [mo.image(Image.open(shot)) for shot in _preview_shots] |
| if _preview_shots |
| else [ |
| mo.md( |
| "_No preview captured yet - run `python preview/capture.py` to " |
| "screenshot the live page._" |
| ) |
| ] |
| ) |
| ) |
| return |
|
|
|
|
| @app.cell |
| def demo(): |
| |
| |
| _detector = OcrDetector() |
| _observation = IndexedScreen(ImageScreen(fake_desktop()), _detector).capture() |
|
|
| perception_rows = [ |
| {"label": e.label, "kind": e.kind, "x": e.x, "y": e.y, "w": e.width, "h": e.height} |
| for e in _observation.elements |
| ] |
|
|
| perception_episode = AgentLoop( |
| screen=IndexedScreen(ImageScreen(fake_desktop()), _detector), |
| brain=PlannerBrain(), |
| executor=DryRunExecutor(), |
| policy=Policy(step_budget=4), |
| ).run("click \'Save\' to save the document") |
|
|
| _clicked = next((s.result.note for s in perception_episode.steps if s.action and s.action.kind == "click"), "n/a") |
|
|
| mo.vstack( |
| [ |
| mo.md( |
| f"**Perception, from pixels.** OCR read **{len(perception_rows)} labels** off a " |
| f"synthetic screenshot; the planner then chose one and the harness resolved it to a real pixel." |
| ), |
| mo.ui.table(perception_rows, selection=None), |
| mo.md(f"`{_clicked}`"), |
| mo.md(f"`{perception_episode.render().splitlines()[1]}`"), |
| ] |
| ) |
| return |
|
|
|
|
| @app.cell |
| def scoreboard(): |
| |
| |
| |
|
|
|
|
| class FirstItemBrain: |
| """A baseline with no model at all: click whatever is first in reading order.""" |
|
|
| name = "first-item" |
|
|
| def decide(self, goal, observation): |
| menu = DecisionMenu(observation) |
| if not menu: |
| return Action("done", reason="nothing to click") |
| return Action("click", label=menu.items[0].label, reason="first item in reading order") |
|
|
|
|
| def _bar(label, x): |
| return Element(label, x=x, y=6, width=80, height=20) |
|
|
|
|
| def _button(label, x, y=270): |
| return Element(label, x=x, y=y, width=90, height=30) |
|
|
|
|
| score_scenarios = [ |
| Scenario( |
| "click 'Save' to save the document", |
| frames=[ |
| (_bar("Files", 0), _bar("Edit", 100), _bar("View", 200)), |
| (Element("Name", x=200, y=195, width=200, height=26), _button("Save", 200)), |
| (_bar("Files", 0), _button("Save", 200)), |
| ], |
| expected="Save", name="save the document", |
| ), |
| Scenario( |
| "click 'Files' to open the file manager", |
| frames=[(_bar("Files", 0), _bar("Edit", 100), _button("Save", 200))], |
| expected="Files", name="open the file manager", |
| ), |
| Scenario( |
| "click 'Cancel' to discard the change", |
| frames=[(_button("Save", 200), _button("Cancel", 400))], |
| expected="Cancel", name="discard the change", |
| ), |
| ] |
|
|
| scoreboard = benchmark( |
| score_scenarios, |
| { |
| "planner (rule-based)": lambda scenario: PlannerBrain(), |
| "first-item (baseline)": lambda scenario: FirstItemBrain(), |
| "oracle (always right)": lambda scenario: MockBrain( |
| [ |
| {"kind": "click", "label": scenario.expected}, |
| {"kind": "done", "reason": "clicked the answer the task names"}, |
| ] |
| ), |
| }, |
| ) |
|
|
| mo.vstack( |
| [ |
| mo.md( |
| f"**Score the choice, offline.** {len(score_scenarios)} tasks x " |
| f"{len(scoreboard.brains())} brains = **{scoreboard.total} decisions**, " |
| f"**{scoreboard.earned} of them right ({scoreboard.accuracy:.0%})** - read off " |
| f"saved element indexes, with no screen, no model and no network." |
| ), |
| mo.ui.table(scoreboard.to_rows(), selection=None), |
| mo.md( |
| "Accuracy per brain: " |
| + " · ".join( |
| f"**{_brain}** {_accuracy:.0%}" |
| for _brain, _accuracy in scoreboard.by_brain().items() |
| ) |
| ), |
| ] |
| ) |
| return |
|
|
|
|
| @app.cell |
| def console_setup(): |
| |
| |
| import tempfile |
|
|
| console_goal = mo.ui.text(value="click 'Save' to save the document", label="goal", |
| full_width=True) |
| console_brain = mo.ui.dropdown( |
| options={"planner (rule-based, no weights)": "planner", "scripted clicks": "scripted"}, |
| value="planner (rule-based, no weights)", label="brain") |
| console_screen = mo.ui.dropdown( |
| options={"replayed screenshot + real OCR": "image", "this machine, live": "live"}, |
| value="replayed screenshot + real OCR", label="screen") |
| console_interval = mo.ui.refresh(options=[5, 10, 30], default_interval=10, |
| label="live preview") |
| console_step = mo.ui.run_button(label="▶ step") |
| console_run = mo.ui.run_button(label="⏩ run", kind="success") |
| console_reset = mo.ui.run_button(label="⟲ reset", kind="warn") |
| console_pause = mo.ui.switch(label="pause", value=False) |
| console_real_input = mo.ui.switch(label="real mouse (needs the allow env var)", value=False) |
| console_panic = mo.ui.switch(label="panic: create the kill switch", value=False) |
|
|
| console_session = { |
| "loop": None, |
| "signature": None, |
| "kill": str(Path(tempfile.gettempdir()) / "tiny-computer-agent.console.STOP"), |
| } |
| return ( |
| console_brain, |
| console_goal, |
| console_interval, |
| console_panic, |
| console_pause, |
| console_real_input, |
| console_reset, |
| console_run, |
| console_screen, |
| console_session, |
| console_step, |
| ) |
|
|
|
|
| @app.cell |
| def console( |
| console_brain, |
| console_goal, |
| console_interval, |
| console_panic, |
| console_pause, |
| console_real_input, |
| console_reset, |
| console_run, |
| console_screen, |
| console_session, |
| console_step, |
| ): |
| |
| from PIL import ImageDraw |
|
|
| _ALLOW_ENV = "TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT" |
| _goal = console_goal.value.strip() or "click 'Save' to save the document" |
| _signature = (_goal, console_brain.value, console_screen.value, |
| bool(console_real_input.value)) |
| _session = console_session |
| _loop = _session["loop"] |
|
|
| |
| if console_panic.value: |
| Path(_session["kill"]).write_text("stop") |
| else: |
| Path(_session["kill"]).unlink(missing_ok=True) |
|
|
| if _loop is None or _session["signature"] != _signature: |
| if console_brain.value == "scripted clicks": |
| _brain = MockBrain([{"kind": "click", "label": "Save"}, |
| {"kind": "done", "reason": "clicked Save"}]) |
| else: |
| _brain = PlannerBrain() |
| if console_screen.value == "this machine, live": |
| _screen = IndexedScreen(GrabScreen(monitor=1), OcrDetector()) |
| else: |
| _screen = IndexedScreen(ImageScreen(fake_desktop()), OcrDetector()) |
| _real = bool(console_real_input.value) and os.environ.get(_ALLOW_ENV) == "1" |
| _loop = AgentLoop( |
| screen=_screen, brain=_brain, |
| executor=PyAutoGuiExecutor() if _real else DryRunExecutor(), |
| policy=Policy(dry_run=not _real, step_budget=8, kill_switch_path=_session["kill"]), |
| ) |
| _loop.begin(_goal) |
| _session.update(loop=_loop, signature=_signature) |
|
|
| |
| |
| _loop.paused = bool(console_pause.value) |
| if console_reset.value: |
| _loop.begin(_goal) |
| if console_step.value: |
| _loop.paused = False |
| _loop.step_once() |
| if console_run.value: |
| _loop.run(_goal, max_steps=8) |
|
|
| |
| _observation = _loop.observation |
| if console_screen.value == "this machine, live": |
| _observation = _loop.screen.capture() |
| _armed = console_interval.value |
|
|
| _preview = None |
| if _observation is not None and hasattr(_observation.frame, "convert"): |
| _preview = _observation.frame.convert("RGB").copy() |
| _pen = ImageDraw.Draw(_preview) |
| for _element in _observation.elements: |
| _pen.rectangle( |
| [_element.x, _element.y, |
| _element.x + max(_element.width, 1), _element.y + max(_element.height, 1)], |
| outline=(255, 138, 0), width=2) |
| _pen.text((_element.x + 3, max(0, _element.y - 12)), _element.label[:30], |
| fill=(210, 90, 0)) |
|
|
| _rows = [{"step": _step.index, |
| "action": _step.action.describe() if _step.action else "", |
| "outcome": _step.error or (_step.result.summary() if _step.result else "")} |
| for _step in _loop.episode.steps] |
|
|
| _state = "finished" if _loop.finished else ("paused" if _loop.paused else "ready") |
| _mode = "REAL mouse" if not _loop.executor.dry_run else "dry run (nothing is clicked)" |
| console_status = mo.md( |
| f"**{_state}** · {len(_rows)} step(s) · {_mode} · budget {_loop.policy.step_budget} · " |
| f"stop: `{_loop.episode.stop_reason}`" |
| ) |
| if console_real_input.value and os.environ.get(_ALLOW_ENV) != "1": |
| console_status = mo.vstack([ |
| console_status, |
| mo.md(f"`{_ALLOW_ENV}` is not set, so this stays a dry run.").callout(kind="warn"), |
| ]) |
|
|
| mo.vstack([ |
| mo.hstack([console_goal, console_brain, console_screen, console_interval], |
| justify="start", wrap=True), |
| mo.hstack([console_step, console_run, console_pause, console_reset, |
| console_real_input, console_panic], justify="start", wrap=True), |
| console_status, |
| mo.image(_preview) if _preview is not None else mo.md("_no frame yet_"), |
| mo.ui.table(_rows, selection=None) if _rows else mo.md("_no actions yet_"), |
| ]) |
| return |
|
|
|
|
| if __name__ == "__main__": |
| app.run() |
|
|
