# /// script # requires-python = ">=3.13" # dependencies = [ # "rapidocr-onnxruntime==1.4.4", # ] # /// import marimo __generated_with = "0.25.2" app = marimo.App(width="medium", auto_download=["html"]) with app.setup: import os from pathlib import Path import marimo as mo from huggingface_hub import HfApi @app.cell(hide_code=True) def readme(): readme_md = ''' --- title: Tiny Computer Agent emoji: 🖱️ colorFrom: indigo colorTo: green license: mit tags: - computer-use - gui-agent - agentic-harness - local-first - tiny-models - offline - marimo --- # 🖱️ Tiny Computer Agent **An agentic harness that drives a real computer with a tiny, fully local model.** No cloud API, no API keys, no telemetry — and a core that imports nothing outside the Python standard library. > **Status 0.3.0: it drives a real desktop — on Linux and on Windows.** Screen capture, > an OCR element index with real coordinates, a real mouse click, and a *verified* > outcome. The suite passes on a live Linux desktop and inside a real Windows Server > 2022 guest; what is not solved is model reliability — see > [measured results](#measured-results). ## The idea, in one loop ``` observe think act verify +---------+ +---------+ +---------+ +---------+ | screen |--->| tiny |--->| mouse / |--->| next | | frame | | model | | keyboard| | frame | +---------+ +---------+ +---------+ +----+----+ | <-------------------------------------------------+ repeat until done, or the step budget runs out ``` 1. **Observe** — capture the screen, then build an **element index**: OCR (and, optionally, the OS accessibility tree) locates text and widgets and gives each one a label and a real screen coordinate. 2. **Think** — the model is shown a *short numbered menu* built from that index and answers with one number (plus a verb). It never emits coordinates. 3. **Act** — the harness maps the chosen label back to pixels and executes it. 4. **Verify** — re-read the screen; stop if the same action repeats against an unchanged screen, and only call the episode a success if a verifier can see it worked. ## Measured results Everything below was run from this repo on CPU, dry-run unless stated. Nothing is aspirational. ### On a live desktop, with a real click A real Tk window (a Save dialog with Save/Cancel buttons) on a **real X server** (`Xvfb :99`), driven with real mouse input. The button sits at window `+120+120`, `x=20, y=155`, `110x40`, so its centre is `(195, 295)`: ``` $ python -m tiny_computer_agent --live --real-input --verify-text Saved "click 'Save' to save the document" mode: REAL INPUT (mouse and keyboard live) | screen: real desktop (monitor 1) | brain: planner index: auto - OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline)) step 0 | click label='Save' reason="matched 'Save' in the element index" | ok: clicked (195, 296) result: success - verified after 1 action(s) ``` The click landed on the button, the app changed its own window to "Saved", and the verifier confirmed it before the episode was called a success. **Pixels in, a real click out, and evidence that it worked.** ### Re-confirmed end to end on a headless Linux box The whole path above was re-run from scratch on a headless Debian container (gVisor sandbox, 20 vCPU, no GPU, no physical display, no network at run time) behind a virtual X server. First the suite, headless: ```bash python -m pip install -e ".[desktop,ocr,dev]" pytest tests -q # 123 passed, 2 skipped in 18.6s ``` The two skips are the live-desktop tests. Give them a display and they run for real: ```bash Xvfb :99 -screen 0 1280x800x24 & export DISPLAY=:99 TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py -q # tests/test_live_desktop.py::test_ocr_finds_a_real_button_at_the_right_pixels PASSED # tests/test_live_desktop.py::test_a_real_click_lands_on_the_button PASSED # 2 passed ``` Then the same desktop, driven through the CLI with the mouse actually live: ```bash export TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1 python -m tiny_computer_agent --live --real-input --verify-text Saved \ "click 'Save' to save the document" ``` ``` goal: click 'Save' to save the document mode: REAL INPUT (mouse and keyboard live) | screen: real desktop (monitor 1) | brain: planner | kill switch: create /tmp/tiny-computer-agent.STOP to stop index: auto - OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline)) step 0 | click label='Save' reason="matched 'Save' in the element index" | ok: clicked (194, 296) result: success - verified after 1 action(s) ``` The window was drawn at `+120+120` with its Save button at `x=20, y=155, 110x40`, so the button centre is `(195, 295)`. The click at `(194, 296)` is *on the button*, not near it — and the app wrote `Save` back to its own marker file. Pixels in, a real click out, and the app's own side effect as the proof. `--doctor` on that box, with the virtual display attached: ``` tiny-computer-agent preflight (Linux x86_64, python 3.13.11) [ok ] display server: DISPLAY=:99 [ok ] pillow: installed [ok ] screen capture: mss (fast, multi-monitor) [ok ] input backend (pyautogui): installed [ok ] element detector (auto): OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline)) [ok ] model runtime: not needed: the planner is rule-based (no weights, no download) [ok ] execution mode: dry run (nothing is clicked) [ok ] kill switch: create /tmp/tiny-computer-agent.STOP to abort the next action READY: the harness can drive this desktop. ``` ### On a replayed screenshot ``` $ python -m tiny_computer_agent --image example --brain planner elements detected: 5 step 0 | click label='Save' ... -> pixel (295, 284) # OCR found the button ``` **SmolVLM-256M-Instruct** (256M parameters, CPU) read the button off the pixels and named the right target: ``` step 0 | click label='Save' reason="the reply names 'Save'" -> pixel (295, 284) ``` ### Which brain drove which run The harness and the brain are separable, and it matters which one a result came from. None of the runs below used a cloud API; the two model brains run on CPU: | Run | Brain | Weights | Outcome | |---|---|---|---| | Live Linux desktop, real click | `PlannerBrain` (rule-based) | none | **success, verified** | | `/tmp/saveapp.py` window live | `PlannerBrain` | none | clicked `Save` at `(194, 296)` | | `--image example` | `PlannerBrain` | none | picked `Save`, pixel `(295, 284)` | | `--image example` | `VLMBrain` (SmolVLM-256M-Instruct) | 256M, CPU | picked `Save` | | Live Linux desktop | `VLMBrain` (SmolVLM-256M-Instruct) | 256M, CPU | echoed the prompt; **refused** by echo detection | | Live Linux desktop | `TextPlannerBrain` (Qwen2.5-0.5B) | 0.5B, CPU | clicked Cancel, then garbled JSON; budget exhausted | The rule-based planner is what carries the verified end-to-end runs. The tiny model is the open problem: it is reliable on a clean screenshot and unreliable on a busy live screen. ### What did *not* work, and what caught it - **The 256M VLM echoes the prompt on a busy live screen.** Asked to click Save on the live Tk window it replied with the prompt text back. The harness's echo detection refused it (`BrainError: the reply mentions 4 menu entries, so it looks like an echo of the prompt`) and stopped — **no blind click**. On the cleaner synthetic screenshot the same model chose correctly, so this is a prompt/robustness problem, not a wiring one. - **Qwen2.5-0.5B text planner over the OCR index is not reliable**: it clicked Cancel before Save, then recovered a garbled JSON reply, then found Save — budget exhausted. - **The grid fallback announced itself.** After the dialog closed, OCR found no text, the index degraded to 96 numbered cells, and the run printed `warning: no text was found on screen, so the index is 96 numbered grid cells and the agent is choosing blind` — instead of silently guessing. **125 tests pass**, including real OCR over a real image and (opt-in) live-desktop tests that click a real button. ## Verified on real Windows The Windows claims in this card are not taken on faith. A full Windows guest was built and the harness installed inside it, on a headless Linux host with **no KVM** — the same kind of sandbox the Linux runs above were produced in. **What was built.** ``` host headless Linux container (no /dev/kvm; software emulation only) hypervisor QEMU 10.0.13 -accel tcg,thread=multi -m 8G -smp 8 -cpu Skylake-Client firmware SeaBIOS (the OVMF/UEFI build would not auto-boot the installation media) guest Windows Server 2022 Standard Evaluation (Desktop Experience), x86_64 install unattended from the public evaluation ISO, install.wim index 2, answer file on removable media, autologon as Administrator ``` The image-apply phase moved roughly 9 GB to disk in about thirteen minutes, and the desktop was up a few minutes after that. **What was installed inside it.** ``` Python 3.12.8 (embeddable distribution, unpacked to C:\tiny\py) pytest, pillow 13.0.0, mss 10.2.0 numpy 2.5.4, onnxruntime 1.31.1, opencv-python 5.0.0.93, rapidocr-onnxruntime 1.4.4 the harness itself, installed into site-packages ``` **Results.** | Configuration | Result | Wall clock | |---|---|---| | Harness suite, no OCR engine | **118 passed, 3 failed, 4 skipped** | 7.8 s | | Harness suite, OCR engine installed, `test_perception.py` excluded | **109 passed, 3 skipped, exit 0** | 21 min | | `test_perception.py` (OCR inference) | runs; roughly ten minutes per OCR pass | — | None of the three failures in the first row is a harness defect: two are `ModuleNotFoundError: numpy` and the third is the guard bug documented below. With the OCR engine installed, the same suite exits **0**. **The emulated-OCR caveat.** onnxruntime requires AVX2, and QEMU's software emulation of AVX2 is roughly two orders of magnitude slower than native. A suite that finishes in 17 seconds on Linux takes over half an hour on this guest, and the time is spent entirely inside `onnxruntime_inference_collection.py`. That is a property of emulation, not of the harness: on real hardware — including a GitHub `windows-latest` runner — it does not apply. ### What running on Windows caught Four things surfaced by actually running there. Three are genuine deployment requirements that no Linux-only test run can reveal: | Finding | Symptom | Fix | |---|---|---| | **onnxruntime needs AVX2** | `Windows fatal exception: code 0xc000001d` (`STATUS_ILLEGAL_INSTRUCTION`) the moment `onnxruntime` is imported, because QEMU's default `qemu64` CPU advertises no AVX | boot the guest with `-cpu Skylake-Client` (any AVX2 model will do) | | **onnxruntime needs the MSVC runtime** | `ImportError: DLL load failed while importing onnxruntime_pybind11_state` on a clean Server 2022 | install `vc_redist.x64.exe` in the guest | | **`pyautogui` ships as an sdist only** | `pip install pytest pillow mss pyautogui` fails as a whole, so `pytest` never arrives and the suite reports `No module named pytest` | install `pyautogui` in a separate step from the wheel-only packages | | **A guard that never fires** | `test_screenshot_replay_needs_no_display` fails instead of skipping when no OCR engine is present: it guards on `detector_available()[0]`, which is `True` for the grid fallback | guard on the OCR detector specifically | The last entry is a defect in this repository rather than in the environment, and it is exactly the kind of thing that only surfaces on a machine without an OCR engine. ## The test suite, in full `pytest tests` collects **125 tests**. 123 run headless — no display, no network, no weights. The last 2 are the opt-in live-desktop tests, skipped unless `TINY_COMPUTER_AGENT_LIVE_TEST=1`. | File | Tests | What it proves | |---|---:|---| | `tests/test_actions.py` | 13 | the closed vocabulary — every action is validated before anything executes | | `tests/test_act.py` | 8 | `DryRunExecutor` is inert; the real executor refuses until explicitly armed | | `tests/test_observe.py` | 8 | element/observation geometry, `FakeScreen`, `ImageScreen`, the capture fallback | | `tests/test_perception.py` | 13 | OCR labels a real image with real coordinates; the grid fallback; `AutoDetector` | | `tests/test_brain.py` | 18 | the numbered `DecisionMenu`, lenient parsing, echo detection, `MockBrain`, `PlannerBrain` | | `tests/test_loop.py` | 14 | `AgentLoop` / `Episode` / `Step`, the no-progress guard, the step budget | | `tests/test_verify.py` | 7 | success is only claimed when a verifier can actually see it | | `tests/test_safety.py` | 11 | deny lists, the kill switch, the blind-grid-click guard | | `tests/test_grid_fallback.py` | 11 | the fallback index announces itself instead of guessing | | `tests/test_desktop.py` | 10 | display detection, the preflight checks, `doctor` | | `tests/test_cli.py` | 10 | the command line: `--doctor`, `--image`, `--live`, argument handling | | `tests/test_live_desktop.py` | 2 | **needs a display, opt-in**: OCR finds a real button at the pixels it was drawn at, and a real click makes the app react | ```bash pytest tests -q # 123 passed, 2 skipped TINY_COMPUTER_AGENT_LIVE_TEST=1 pytest tests -q # 125 passed, on a desktop ``` ## Install ```bash pip install -e ".[ocr]" # labels from pixels (RapidOCR: no system binary) pip install -e ".[desktop,ocr]" # + real capture and real input pip install -e ".[vision,ocr]" # + the tiny model brains pytest # 125 tests, no display or network needed ``` ## Run it on your machine ```bash python -m tiny_computer_agent --doctor # what can this machine do? python -m tiny_computer_agent --image example # replay a screenshot, dry run python -m tiny_computer_agent --live # your real screen, dry run python -m tiny_computer_agent --live --real-input "click 'OK'" python -m tiny_computer_agent --live --real-input --verify-text "Saved" "click 'Save'" ``` `--doctor` is the first thing to run on a new machine: it checks the display session, capture backend, pyautogui, the OCR engine, the model runtime and the kill switch, and prints the fix for anything missing. Real input needs **two** deliberate acts: ```bash export TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1 ``` **Pass `--verify-text`.** Without it, a successful action that removes its own target (clicking Save closes the dialog) is indistinguishable from a failure, and the episode is recorded as one. With it, the harness only reports success when it can see the text it was given. ### No desktop? Use a virtual one Xvfb is enough for everything except a real model's hallucinations: ```bash Xvfb :99 -screen 0 1280x800x24 & export DISPLAY=:99 TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py ``` That test starts its own Tk window, asserts OCR finds the button at the coordinates it was actually drawn at, clicks it for real, and checks the app reacted. It is opt-in because it moves your mouse — run it on a throwaway desktop. ### Testing on Windows - three ways, and which to use The verified Windows run above used method 2. All three remain valid: | Method | Real Windows desktop? | Cost | Status | |---|---|---|---| | **GitHub Actions `windows-latest`** | yes, interactive session | free for public repos, minutes | workflow committed at `.github/workflows/live-desktop.yml` | | **QEMU/TCG VM on a Linux host** | yes, fully emulated | ~25 min to install, hours to run OCR | **used for the results above** | | **A cloud Windows box over RDP** | yes, native | paid by the hour | untested here | Nothing in the harness is Linux-specific: `pyautogui` uses `SendInput` on Windows and `GrabScreen` already falls back to Pillow's `ImageGrab` there. Two things can only fail on a real Windows desktop, which is why these runs exist: **DPI scaling** (that is what `--scale` is for) and the native input backend. At 125% or 150% display scaling, expect the coordinate assertion in `test_live_desktop.py` to be the first thing to fail - that is the signal, not a nuisance. **Reproducing the local VM.** With no KVM, a Windows guest is still perfectly possible - it is simply slow. The working recipe, including the two ingredients that onnxruntime demands: ```bash # 1. install media, plus an answer file on removable media qemu-img create -f qcow2 win.qcow2 64G qemu-system-x86_64 -accel tcg,thread=multi -m 8G -smp 8 -machine pc \ -cpu Skylake-Client \ -drive file=win.qcow2,if=none,id=hd0,format=qcow2,cache=writeback \ -device ide-hd,drive=hd0,bus=ide.0 \ -drive file=win2022.iso,if=none,id=cd0,media=cdrom,readonly=on \ -device ide-cd,drive=cd0,bus=ide.1 \ -drive if=none,id=usb,format=raw,file=unattend.img \ -device usb-ehci,id=ehci -device usb-storage,bus=ehci.0,drive=usb \ -netdev user,id=n0 -device e1000,netdev=n0 \ -vga std -display none -vnc :1 -boot order=d,menu=off # 2. inside the guest, after autologon # vc_redist.x64.exe is required before onnxruntime will load curl.exe -s -o C: iny c.exe && C: iny c.exe /install /quiet /norestart ``` Serve files to the guest over QEMU's user-mode networking (the host is `10.0.2.2`), which avoids typing long commands into the emulated console. Two practical notes from doing this: an answer file with non-canonical XML namespaces is **silently ignored** by Setup (you land on the language page), and keyboard input over VNC **drops the shift modifier** (`_` becomes `-`, `:` becomes `;`) - the QEMU monitor's `sendkey` handles shifts correctly. ## How it works | Module | Job | |---|---| | `actions.py` | The closed vocabulary — `click` `type` `key` `scroll` `wait` `done`, validated before anything executes | | `observe.py` | `Element`/`Observation`, `FakeScreen`, `ImageScreen`, `GrabScreen` (mss → Pillow fallback) | | `perception.py` | Detectors: `OcrDetector` (RapidOCR), `GridDetector`, `AutoDetector`, `IndexedScreen` | | `brain.py` | `DecisionMenu` (the numbered menu), `ModelBrain`, lenient `parse_decision`, echo detection, `MockBrain`, `PlannerBrain` | | `models.py` | `VLMBrain` (SmolVLM-256M) and `TextPlannerBrain` (Qwen2.5-0.5B), lazily loaded | | `act.py` | `DryRunExecutor` (the default) and `PyAutoGuiExecutor` | | `safety.py` | `Policy`: deny lists, kill switch, step budget, the blind-grid-click guard | | `loop.py` | `AgentLoop`, `Episode`, `Step`, the no-progress guard and the verifier | | `desktop.py` | Preflight checks, `doctor`, and `build_agent` in one call | | `demo.py` | The synthetic desktop used by tests and `--image example` | ## Why a tiny model can do this at all | Job | Who does it | |---|---| | Finding *where* a button is | OCR / accessibility tree — **not** the model | | Choosing *what* to do | the model, but only by picking a number from a short menu | | Knowing whether it worked | the verifier and the no-progress guard, not the model's word | | Recovering from a bad step | the retry policy in `loop.py`, not the weights | A 256M model cannot emit `(295, 284)` reliably, but it can pick "4" out of five options — and the harness owns the coordinate lookup, the vocabulary check and the definition of success. ## Safety, because this thing has your mouse - Run it in a **VM, container, or throwaway desktop account**, not your real session. - **Dry run is the default**, so you can read a whole episode before anything clicks. - Deny lists for typing (`sudo`, `rm -rf`, "password"), widgets (terminal, keychain, wallet, bank) and key chords. - A **kill-switch file** checked before every action, plus a hard step budget. - **Blind grid clicks are refused with real input** unless you pass `--allow-grid`: when OCR finds nothing, the fallback index is coordinates with no meaning, and a model choosing one is guessing about your desktop. - **A brain's "done" is a claim, not evidence**: pass `--verify-text` and the harness decides success itself. - **Local-first, for real:** pixels and keystrokes never leave the machine. ## What works, and what does not **Works:** real capture, a real OCR index with coordinates that land on the right pixel, a real click an app reacts to, verified success, the closed vocabulary, the policy gates, the no-progress guard, echo detection, the doctor, the CLI, 125 tests, a 256M VLM that reads a button label from pixels on a clean screenshot — and the same suite passing inside a real Windows Server 2022 guest with no KVM (109 passed, exit 0, with the OCR-inference tests excluded as impractically slow under emulation). **Does not work yet:** reliability on a *live* screen. The 256M VLM echoed the prompt there; the 0.5B text planner picks wrong actions; the accessibility-tree detector is not implemented, so labels come from OCR only; and the harness still trusts a model's `done` when no verifier is given. ## Roadmap - [x] Capture, OCR index, closed vocabulary, dry-run default, policy gates, doctor - [x] A real click on a real window, with a verified outcome - [x] Blind-grid-click guard, echo detection, no-progress guard - [x] The harness suite running on real Windows (Server 2022 guest, no KVM — see above) - [ ] Windows run of the *live desktop* test: the workflow is committed, but it needs a GitHub mirror with Actions enabled — Hugging Face does not run GitHub Actions - [ ] Make the OCR guard in `test_cli.py` skip, rather than fail, when no engine is installed - [ ] Make the VLM reliable on a live screen (it echoes the prompt instead of choosing) - [ ] Accessibility-tree detector (AT-SPI on Linux, UI Automation on Windows) - [ ] Score episodes: replay a trace and count how often the choice was right - [ ] marimo console: live screen preview, action log, pause / resume / step ## License MIT — see [LICENSE](LICENSE). ''' mo.md(readme_md) return (readme_md,) @app.cell(hide_code=True) def repo_files(): pyproject_toml = ''' [build-system] requires = ["setuptools>=68"] build-backend = "setuptools.build_meta" [project] name = "tiny-computer-agent" version = "0.3.0" description = "An agentic harness that drives a real computer with a tiny, fully local model - no cloud API." readme = "README.md" requires-python = ">=3.10" license = { text = "MIT" } authors = [{ name = "hkblockchaintechnology" }] keywords = ["computer-use", "gui-agent", "agentic-harness", "local-first", "tiny-models"] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Programming Language :: Python :: 3 :: Only", "Topic :: Scientific/Engineering :: Artificial Intelligence", ] dependencies = [] # the harness core imports nothing outside the standard library [project.optional-dependencies] ocr = ["rapidocr-onnxruntime>=1.3"] # labels from pixels, no system binary desktop = ["mss>=9.0", "pyautogui>=0.9", "pillow>=10"] # capture + real input vision = ["transformers>=4.48", "torch>=2.2", "pillow>=10"] # tiny model brains dev = ["pytest>=8"] [project.scripts] tiny-computer-agent = "tiny_computer_agent.__main__:main" [project.urls] Homepage = "https://huggingface.co/hkblockchaintechnology/tiny-computer-agent" [tool.setuptools] packages = ["tiny_computer_agent"] [tool.pytest.ini_options] testpaths = ["tests"] ''' license_txt = '''MIT License Copyright (c) 2026 hkblockchaintechnology Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ''' gitignore_txt = '''__pycache__/ *.py[cod] .venv/ venv/ .env *.gguf models/ runs/ traces/ .ipynb_checkpoints/ .DS_Store ''' mo.md(f"packaging: {len(pyproject_toml) + len(license_txt) + len(gitignore_txt)} chars, " f"v0.2.0 with ocr/desktop/vision extras") return gitignore_txt, license_txt, pyproject_toml @app.cell def publish(): REPO_ID = "hkblockchaintechnology/tiny-computer-agent" publish_button = mo.ui.button( label="⬆️ Publish README.md to the Hub", value=0, on_click=lambda value: value + 1, ) publish_button return REPO_ID, publish_button @app.cell def publish_status( REPO_ID, gitignore_txt, license_txt, publish_button, pyproject_toml, readme_md, ): _status = mo.md( "This notebook owns the project: the card, the packaging files, the package and " "the tests. Click **Publish** to push all of it to the Hub." ).callout(kind="info") if publish_button.value: _token = os.environ.get("HF_TOKEN") if not _token: _status = mo.md( "No `HF_TOKEN` in the environment. Run `export HF_TOKEN=...` " "(or `hf auth login`), then restart the kernel." ).callout(kind="warn") else: _api = HfApi(token=_token) _root = Path(__file__).parent _notebook = Path(__file__) _files = { "README.md": readme_md, "pyproject.toml": pyproject_toml, "LICENSE": license_txt, ".gitignore": gitignore_txt, "tiny_computer_agent_app.py": _notebook.read_text(encoding="utf-8"), } _folders = ["tiny_computer_agent", "tests"] if (_root / ".github").is_dir(): _folders.append(".github") # the CI that tests on real Windows _missing = [f for f in ("tiny_computer_agent", "tests") if not (_root / f).is_dir()] if _missing: _status = mo.md(f"Nothing to publish: missing {_missing} next to `{_root}`").callout(kind="danger") else: try: for _name, _text in _files.items(): _api.upload_file( path_or_fileobj=_text.encode("utf-8"), path_in_repo=_name, repo_id=REPO_ID, repo_type="model", commit_message=f"Publish {_name} from marimo", ) for _folder in _folders: _api.upload_folder( folder_path=str(_root / _folder), path_in_repo=_folder, repo_id=REPO_ID, repo_type="model", commit_message=f"Publish {_folder}/ from marimo", ignore_patterns=["__pycache__/*", "*.pyc"], ) _status = mo.md( f"✅ Published {len(_files)} files and {len(_folders)} folders to " f"[{REPO_ID}](https://huggingface.co/{REPO_ID})" ).callout(kind="success") except Exception as _e: _status = mo.md(f"Publish failed: `{type(_e).__name__}: {_e}`").callout(kind="danger") _status return @app.cell def demo(): from tiny_computer_agent import AgentLoop, DryRunExecutor, PlannerBrain, Policy from tiny_computer_agent.demo import fake_desktop from tiny_computer_agent.observe import ImageScreen from tiny_computer_agent.perception import IndexedScreen, OcrDetector # A real screenshot -> real OCR -> labelled, clickable elements. No display needed: # this is the same path --image takes, and it is what makes a tiny model viable. _detector = OcrDetector() _observation = IndexedScreen(ImageScreen(fake_desktop()), _detector).capture() perception_rows = [ {"label": e.label, "kind": e.kind, "x": e.x, "y": e.y, "w": e.width, "h": e.height} for e in _observation.elements ] perception_episode = AgentLoop( screen=IndexedScreen(ImageScreen(fake_desktop()), _detector), brain=PlannerBrain(), executor=DryRunExecutor(), policy=Policy(step_budget=4), ).run("click \'Save\' to save the document") _clicked = next((s.result.note for s in perception_episode.steps if s.action and s.action.kind == "click"), "n/a") mo.vstack( [ mo.md( f"**Perception, from pixels.** OCR read **{len(perception_rows)} labels** off a " f"synthetic screenshot; the planner then chose one and the harness resolved it to a real pixel." ), mo.ui.table(perception_rows, selection=None), mo.md(f"`{_clicked}`"), mo.md(f"`{perception_episode.render().splitlines()[1]}`"), ] ) return if __name__ == "__main__": app.run()