| |
| |
| |
| |
| |
| |
|
|
| import marimo |
|
|
| __generated_with = "0.25.2" |
| app = marimo.App(width="medium", auto_download=["html"]) |
|
|
| with app.setup: |
| import os |
| from pathlib import Path |
|
|
| import marimo as mo |
| from huggingface_hub import HfApi |
|
|
|
|
| @app.cell(hide_code=True) |
| def readme(): |
| readme_md = ''' |
| |
| --- |
| title: Tiny Computer Agent |
| emoji: 🖱️ |
| colorFrom: indigo |
| colorTo: green |
| license: mit |
| tags: |
| - computer-use |
| - gui-agent |
| - agentic-harness |
| - local-first |
| - tiny-models |
| - offline |
| - marimo |
| --- |
| |
| # 🖱️ Tiny Computer Agent |
| |
| **An agentic harness that drives a real computer with a tiny, fully local model.** |
| No cloud API, no API keys, no telemetry — and a core that imports nothing outside the |
| Python standard library. |
| |
| > **Status 0.3.0: it drives a real desktop — on Linux and on Windows.** Screen capture, |
| > an OCR element index with real coordinates, a real mouse click, and a *verified* |
| > outcome. The suite passes on a live Linux desktop and inside a real Windows Server |
| > 2022 guest; what is not solved is model reliability — see |
| > [measured results](#measured-results). |
| |
| ## The idea, in one loop |
| |
| ``` |
| observe think act verify |
| +---------+ +---------+ +---------+ +---------+ |
| | screen |--->| tiny |--->| mouse / |--->| next | |
| | frame | | model | | keyboard| | frame | |
| +---------+ +---------+ +---------+ +----+----+ |
| | |
| <-------------------------------------------------+ |
| repeat until done, or the step budget runs out |
| ``` |
| |
| 1. **Observe** — capture the screen, then build an **element index**: OCR (and, |
| optionally, the OS accessibility tree) locates text and widgets and gives each one a |
| label and a real screen coordinate. |
| 2. **Think** — the model is shown a *short numbered menu* built from that index and |
| answers with one number (plus a verb). It never emits coordinates. |
| 3. **Act** — the harness maps the chosen label back to pixels and executes it. |
| 4. **Verify** — re-read the screen; stop if the same action repeats against an unchanged |
| screen, and only call the episode a success if a verifier can see it worked. |
| |
| ## Measured results |
| |
| Everything below was run from this repo on CPU, dry-run unless stated. Nothing is |
| aspirational. |
| |
| ### On a live desktop, with a real click |
| |
| A real Tk window (a Save dialog with Save/Cancel buttons) on a **real X server** |
| (`Xvfb :99`), driven with real mouse input. The button sits at window `+120+120`, |
| `x=20, y=155`, `110x40`, so its centre is `(195, 295)`: |
| |
| ``` |
| $ python -m tiny_computer_agent --live --real-input --verify-text Saved "click 'Save' to save the document" |
| mode: REAL INPUT (mouse and keyboard live) | screen: real desktop (monitor 1) | brain: planner |
| index: auto - OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline)) |
| |
| step 0 | click label='Save' reason="matched 'Save' in the element index" | ok: clicked (195, 296) |
| |
| result: success - verified after 1 action(s) |
| ``` |
| |
| The click landed on the button, the app changed its own window to "Saved", and the |
| verifier confirmed it before the episode was called a success. **Pixels in, a real click |
| out, and evidence that it worked.** |
| |
| ### Re-confirmed end to end on a headless Linux box |
| |
| The whole path above was re-run from scratch on a headless Debian container (gVisor |
| sandbox, 20 vCPU, no GPU, no physical display, no network at run time) behind a virtual |
| X server. First the suite, headless: |
| |
| ```bash |
| python -m pip install -e ".[desktop,ocr,dev]" |
| pytest tests -q |
| # 123 passed, 2 skipped in 18.6s |
| ``` |
| |
| The two skips are the live-desktop tests. Give them a display and they run for real: |
| |
| ```bash |
| Xvfb :99 -screen 0 1280x800x24 & |
| export DISPLAY=:99 |
| TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py -q |
| # tests/test_live_desktop.py::test_ocr_finds_a_real_button_at_the_right_pixels PASSED |
| # tests/test_live_desktop.py::test_a_real_click_lands_on_the_button PASSED |
| # 2 passed |
| ``` |
| |
| Then the same desktop, driven through the CLI with the mouse actually live: |
| |
| ```bash |
| export TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1 |
| python -m tiny_computer_agent --live --real-input --verify-text Saved \ |
| "click 'Save' to save the document" |
| ``` |
| |
| ``` |
| goal: click 'Save' to save the document |
| mode: REAL INPUT (mouse and keyboard live) | screen: real desktop (monitor 1) | brain: planner | kill switch: create /tmp/tiny-computer-agent.STOP to stop |
| index: auto - OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline)) |
| |
| step 0 | click label='Save' reason="matched 'Save' in the element index" | ok: clicked (194, 296) |
| |
| result: success - verified after 1 action(s) |
| ``` |
| |
| The window was drawn at `+120+120` with its Save button at `x=20, y=155, 110x40`, so the |
| button centre is `(195, 295)`. The click at `(194, 296)` is *on the button*, not near it — |
| and the app wrote `Save` back to its own marker file. Pixels in, a real click out, and the |
| app's own side effect as the proof. |
| |
| `--doctor` on that box, with the virtual display attached: |
| |
| ``` |
| tiny-computer-agent preflight (Linux x86_64, python 3.13.11) |
| |
| [ok ] display server: DISPLAY=:99 |
| [ok ] pillow: installed |
| [ok ] screen capture: mss (fast, multi-monitor) |
| [ok ] input backend (pyautogui): installed |
| [ok ] element detector (auto): OCR when it finds text, grid otherwise (rapidocr-onnxruntime ready (CPU, offline)) |
| [ok ] model runtime: not needed: the planner is rule-based (no weights, no download) |
| [ok ] execution mode: dry run (nothing is clicked) |
| [ok ] kill switch: create /tmp/tiny-computer-agent.STOP to abort the next action |
| |
| READY: the harness can drive this desktop. |
| ``` |
| |
| |
| ### On a replayed screenshot |
| |
| ``` |
| $ python -m tiny_computer_agent --image example --brain planner |
| elements detected: 5 |
| step 0 | click label='Save' ... -> pixel (295, 284) # OCR found the button |
| ``` |
| |
| **SmolVLM-256M-Instruct** (256M parameters, CPU) read the button off the pixels and |
| named the right target: |
| |
| ``` |
| step 0 | click label='Save' reason="the reply names 'Save'" -> pixel (295, 284) |
| ``` |
| |
| ### Which brain drove which run |
| |
| The harness and the brain are separable, and it matters which one a result came from. |
| None of the runs below used a cloud API; the two model brains run on CPU: |
| |
| | Run | Brain | Weights | Outcome | |
| |---|---|---|---| |
| | Live Linux desktop, real click | `PlannerBrain` (rule-based) | none | **success, verified** | |
| | `/tmp/saveapp.py` window live | `PlannerBrain` | none | clicked `Save` at `(194, 296)` | |
| | `--image example` | `PlannerBrain` | none | picked `Save`, pixel `(295, 284)` | |
| | `--image example` | `VLMBrain` (SmolVLM-256M-Instruct) | 256M, CPU | picked `Save` | |
| | Live Linux desktop | `VLMBrain` (SmolVLM-256M-Instruct) | 256M, CPU | echoed the prompt; **refused** by echo detection | |
| | Live Linux desktop | `TextPlannerBrain` (Qwen2.5-0.5B) | 0.5B, CPU | clicked Cancel, then garbled JSON; budget exhausted | |
| |
| The rule-based planner is what carries the verified end-to-end runs. The tiny model is the |
| open problem: it is reliable on a clean screenshot and unreliable on a busy live screen. |
| |
| ### What did *not* work, and what caught it |
| |
| - **The 256M VLM echoes the prompt on a busy live screen.** Asked to click Save on the |
| live Tk window it replied with the prompt text back. The harness's echo detection |
| refused it (`BrainError: the reply mentions 4 menu entries, so it looks like an echo of |
| the prompt`) and stopped — **no blind click**. On the cleaner synthetic screenshot the |
| same model chose correctly, so this is a prompt/robustness problem, not a wiring one. |
| - **Qwen2.5-0.5B text planner over the OCR index is not reliable**: it clicked Cancel |
| before Save, then recovered a garbled JSON reply, then found Save — budget exhausted. |
| - **The grid fallback announced itself.** After the dialog closed, OCR found no text, the |
| index degraded to 96 numbered cells, and the run printed |
| `warning: no text was found on screen, so the index is 96 numbered grid cells and the |
| agent is choosing blind` — instead of silently guessing. |
| |
| **125 tests pass**, including real OCR over a real image and (opt-in) live-desktop tests |
| that click a real button. |
| |
| ## Verified on real Windows |
| |
| The Windows claims in this card are not taken on faith. A full Windows guest was built |
| and the harness installed inside it, on a headless Linux host with **no KVM** — the same |
| kind of sandbox the Linux runs above were produced in. |
| |
| **What was built.** |
| |
| ``` |
| host headless Linux container (no /dev/kvm; software emulation only) |
| hypervisor QEMU 10.0.13 -accel tcg,thread=multi -m 8G -smp 8 -cpu Skylake-Client |
| firmware SeaBIOS (the OVMF/UEFI build would not auto-boot the installation media) |
| guest Windows Server 2022 Standard Evaluation (Desktop Experience), x86_64 |
| install unattended from the public evaluation ISO, install.wim index 2, |
| answer file on removable media, autologon as Administrator |
| ``` |
| |
| The image-apply phase moved roughly 9 GB to disk in about thirteen minutes, and the |
| desktop was up a few minutes after that. |
| |
| **What was installed inside it.** |
| |
| ``` |
| Python 3.12.8 (embeddable distribution, unpacked to C:\tiny\py) |
| pytest, pillow 13.0.0, mss 10.2.0 |
| numpy 2.5.4, onnxruntime 1.31.1, opencv-python 5.0.0.93, rapidocr-onnxruntime 1.4.4 |
| the harness itself, installed into site-packages |
| ``` |
| |
| **Results.** |
| |
| | Configuration | Result | Wall clock | |
| |---|---|---| |
| | Harness suite, no OCR engine | **118 passed, 3 failed, 4 skipped** | 7.8 s | |
| | Harness suite, OCR engine installed, `test_perception.py` excluded | **109 passed, 3 skipped, exit 0** | 21 min | |
| | `test_perception.py` (OCR inference) | runs; roughly ten minutes per OCR pass | — | |
| |
| None of the three failures in the first row is a harness defect: two are |
| `ModuleNotFoundError: numpy` and the third is the guard bug documented below. With the |
| OCR engine installed, the same suite exits **0**. |
| |
| **The emulated-OCR caveat.** onnxruntime requires AVX2, and QEMU's software emulation of |
| AVX2 is roughly two orders of magnitude slower than native. A suite that finishes in 17 |
| seconds on Linux takes over half an hour on this guest, and the time is spent entirely |
| inside `onnxruntime_inference_collection.py`. That is a property of emulation, not of the |
| harness: on real hardware — including a GitHub `windows-latest` runner — it does not |
| apply. |
| |
| ### What running on Windows caught |
| |
| Four things surfaced by actually running there. Three are genuine deployment requirements |
| that no Linux-only test run can reveal: |
| |
| | Finding | Symptom | Fix | |
| |---|---|---| |
| | **onnxruntime needs AVX2** | `Windows fatal exception: code 0xc000001d` (`STATUS_ILLEGAL_INSTRUCTION`) the moment `onnxruntime` is imported, because QEMU's default `qemu64` CPU advertises no AVX | boot the guest with `-cpu Skylake-Client` (any AVX2 model will do) | |
| | **onnxruntime needs the MSVC runtime** | `ImportError: DLL load failed while importing onnxruntime_pybind11_state` on a clean Server 2022 | install `vc_redist.x64.exe` in the guest | |
| | **`pyautogui` ships as an sdist only** | `pip install pytest pillow mss pyautogui` fails as a whole, so `pytest` never arrives and the suite reports `No module named pytest` | install `pyautogui` in a separate step from the wheel-only packages | |
| | **A guard that never fires** | `test_screenshot_replay_needs_no_display` fails instead of skipping when no OCR engine is present: it guards on `detector_available()[0]`, which is `True` for the grid fallback | guard on the OCR detector specifically | |
| |
| The last entry is a defect in this repository rather than in the environment, and it is |
| exactly the kind of thing that only surfaces on a machine without an OCR engine. |
| |
| ## The test suite, in full |
| |
| `pytest tests` collects **125 tests**. 123 run headless — no display, no network, no |
| weights. The last 2 are the opt-in live-desktop tests, skipped unless |
| `TINY_COMPUTER_AGENT_LIVE_TEST=1`. |
| |
| | File | Tests | What it proves | |
| |---|---:|---| |
| | `tests/test_actions.py` | 13 | the closed vocabulary — every action is validated before anything executes | |
| | `tests/test_act.py` | 8 | `DryRunExecutor` is inert; the real executor refuses until explicitly armed | |
| | `tests/test_observe.py` | 8 | element/observation geometry, `FakeScreen`, `ImageScreen`, the capture fallback | |
| | `tests/test_perception.py` | 13 | OCR labels a real image with real coordinates; the grid fallback; `AutoDetector` | |
| | `tests/test_brain.py` | 18 | the numbered `DecisionMenu`, lenient parsing, echo detection, `MockBrain`, `PlannerBrain` | |
| | `tests/test_loop.py` | 14 | `AgentLoop` / `Episode` / `Step`, the no-progress guard, the step budget | |
| | `tests/test_verify.py` | 7 | success is only claimed when a verifier can actually see it | |
| | `tests/test_safety.py` | 11 | deny lists, the kill switch, the blind-grid-click guard | |
| | `tests/test_grid_fallback.py` | 11 | the fallback index announces itself instead of guessing | |
| | `tests/test_desktop.py` | 10 | display detection, the preflight checks, `doctor` | |
| | `tests/test_cli.py` | 10 | the command line: `--doctor`, `--image`, `--live`, argument handling | |
| | `tests/test_live_desktop.py` | 2 | **needs a display, opt-in**: OCR finds a real button at the pixels it was drawn at, and a real click makes the app react | |
| |
| ```bash |
| pytest tests -q # 123 passed, 2 skipped |
| TINY_COMPUTER_AGENT_LIVE_TEST=1 pytest tests -q # 125 passed, on a desktop |
| ``` |
| |
| ## Install |
| |
| ```bash |
| pip install -e ".[ocr]" # labels from pixels (RapidOCR: no system binary) |
| pip install -e ".[desktop,ocr]" # + real capture and real input |
| pip install -e ".[vision,ocr]" # + the tiny model brains |
| pytest # 125 tests, no display or network needed |
| ``` |
| |
| ## Run it on your machine |
| |
| ```bash |
| python -m tiny_computer_agent --doctor # what can this machine do? |
| python -m tiny_computer_agent --image example # replay a screenshot, dry run |
| python -m tiny_computer_agent --live # your real screen, dry run |
| python -m tiny_computer_agent --live --real-input "click 'OK'" |
| python -m tiny_computer_agent --live --real-input --verify-text "Saved" "click 'Save'" |
| ``` |
| |
| `--doctor` is the first thing to run on a new machine: it checks the display session, |
| capture backend, pyautogui, the OCR engine, the model runtime and the kill switch, and |
| prints the fix for anything missing. Real input needs **two** deliberate acts: |
| |
| ```bash |
| export TINY_COMPUTER_AGENT_ALLOW_REAL_INPUT=1 |
| ``` |
| |
| **Pass `--verify-text`.** Without it, a successful action that removes its own target |
| (clicking Save closes the dialog) is indistinguishable from a failure, and the episode is |
| recorded as one. With it, the harness only reports success when it can see the text it |
| was given. |
| |
| ### No desktop? Use a virtual one |
| |
| Xvfb is enough for everything except a real model's hallucinations: |
| |
| ```bash |
| Xvfb :99 -screen 0 1280x800x24 & |
| export DISPLAY=:99 |
| TINY_COMPUTER_AGENT_LIVE_TEST=1 python -m pytest tests/test_live_desktop.py |
| ``` |
| |
| That test starts its own Tk window, asserts OCR finds the button at the coordinates it |
| was actually drawn at, clicks it for real, and checks the app reacted. It is opt-in |
| because it moves your mouse — run it on a throwaway desktop. |
| |
| ### Testing on Windows - three ways, and which to use |
| |
| The verified Windows run above used method 2. All three remain valid: |
| |
| | Method | Real Windows desktop? | Cost | Status | |
| |---|---|---|---| |
| | **GitHub Actions `windows-latest`** | yes, interactive session | free for public repos, minutes | workflow committed at `.github/workflows/live-desktop.yml` | |
| | **QEMU/TCG VM on a Linux host** | yes, fully emulated | ~25 min to install, hours to run OCR | **used for the results above** | |
| | **A cloud Windows box over RDP** | yes, native | paid by the hour | untested here | |
| |
| Nothing in the harness is Linux-specific: `pyautogui` uses `SendInput` on Windows and |
| `GrabScreen` already falls back to Pillow's `ImageGrab` there. Two things can only fail on |
| a real Windows desktop, which is why these runs exist: **DPI scaling** (that is what |
| `--scale` is for) and the native input backend. At 125% or 150% display scaling, expect the |
| coordinate assertion in `test_live_desktop.py` to be the first thing to fail - that is the |
| signal, not a nuisance. |
| |
| **Reproducing the local VM.** With no KVM, a Windows guest is still perfectly possible - |
| it is simply slow. The working recipe, including the two ingredients that onnxruntime |
| demands: |
| |
| ```bash |
| # 1. install media, plus an answer file on removable media |
| qemu-img create -f qcow2 win.qcow2 64G |
| qemu-system-x86_64 -accel tcg,thread=multi -m 8G -smp 8 -machine pc \ |
| -cpu Skylake-Client \ |
| -drive file=win.qcow2,if=none,id=hd0,format=qcow2,cache=writeback \ |
| -device ide-hd,drive=hd0,bus=ide.0 \ |
| -drive file=win2022.iso,if=none,id=cd0,media=cdrom,readonly=on \ |
| -device ide-cd,drive=cd0,bus=ide.1 \ |
| -drive if=none,id=usb,format=raw,file=unattend.img \ |
| -device usb-ehci,id=ehci -device usb-storage,bus=ehci.0,drive=usb \ |
| -netdev user,id=n0 -device e1000,netdev=n0 \ |
| -vga std -display none -vnc :1 -boot order=d,menu=off |
| |
| # 2. inside the guest, after autologon |
| # vc_redist.x64.exe is required before onnxruntime will load |
| curl.exe -s -o C: iny c.exe <vc_redist-url> && C: iny c.exe /install /quiet /norestart |
| ``` |
| |
| Serve files to the guest over QEMU's user-mode networking (the host is `10.0.2.2`), which |
| avoids typing long commands into the emulated console. Two practical notes from doing |
| this: an answer file with non-canonical XML namespaces is **silently ignored** by Setup |
| (you land on the language page), and keyboard input over VNC **drops the shift modifier** |
| (`_` becomes `-`, `:` becomes `;`) - the QEMU monitor's `sendkey` handles shifts correctly. |
| |
| |
| ## How it works |
| |
| | Module | Job | |
| |---|---| |
| | `actions.py` | The closed vocabulary — `click` `type` `key` `scroll` `wait` `done`, validated before anything executes | |
| | `observe.py` | `Element`/`Observation`, `FakeScreen`, `ImageScreen`, `GrabScreen` (mss → Pillow fallback) | |
| | `perception.py` | Detectors: `OcrDetector` (RapidOCR), `GridDetector`, `AutoDetector`, `IndexedScreen` | |
| | `brain.py` | `DecisionMenu` (the numbered menu), `ModelBrain`, lenient `parse_decision`, echo detection, `MockBrain`, `PlannerBrain` | |
| | `models.py` | `VLMBrain` (SmolVLM-256M) and `TextPlannerBrain` (Qwen2.5-0.5B), lazily loaded | |
| | `act.py` | `DryRunExecutor` (the default) and `PyAutoGuiExecutor` | |
| | `safety.py` | `Policy`: deny lists, kill switch, step budget, the blind-grid-click guard | |
| | `loop.py` | `AgentLoop`, `Episode`, `Step`, the no-progress guard and the verifier | |
| | `desktop.py` | Preflight checks, `doctor`, and `build_agent` in one call | |
| | `demo.py` | The synthetic desktop used by tests and `--image example` | |
| |
| ## Why a tiny model can do this at all |
| |
| | Job | Who does it | |
| |---|---| |
| | Finding *where* a button is | OCR / accessibility tree — **not** the model | |
| | Choosing *what* to do | the model, but only by picking a number from a short menu | |
| | Knowing whether it worked | the verifier and the no-progress guard, not the model's word | |
| | Recovering from a bad step | the retry policy in `loop.py`, not the weights | |
| |
| A 256M model cannot emit `(295, 284)` reliably, but it can pick "4" out of five options — |
| and the harness owns the coordinate lookup, the vocabulary check and the definition of |
| success. |
| |
| ## Safety, because this thing has your mouse |
| |
| - Run it in a **VM, container, or throwaway desktop account**, not your real session. |
| - **Dry run is the default**, so you can read a whole episode before anything clicks. |
| - Deny lists for typing (`sudo`, `rm -rf`, "password"), widgets (terminal, keychain, |
| wallet, bank) and key chords. |
| - A **kill-switch file** checked before every action, plus a hard step budget. |
| - **Blind grid clicks are refused with real input** unless you pass `--allow-grid`: when |
| OCR finds nothing, the fallback index is coordinates with no meaning, and a model |
| choosing one is guessing about your desktop. |
| - **A brain's "done" is a claim, not evidence**: pass `--verify-text` and the harness |
| decides success itself. |
| - **Local-first, for real:** pixels and keystrokes never leave the machine. |
| |
| ## What works, and what does not |
| |
| **Works:** real capture, a real OCR index with coordinates that land on the right pixel, |
| a real click an app reacts to, verified success, the closed vocabulary, the policy gates, |
| the no-progress guard, echo detection, the doctor, the CLI, 125 tests, a 256M VLM that |
| reads a button label from pixels on a clean screenshot — and the same suite passing inside |
| a real Windows Server 2022 guest with no KVM (109 passed, exit 0, with the OCR-inference |
| tests excluded as impractically slow under emulation). |
| |
| **Does not work yet:** reliability on a *live* screen. The 256M VLM echoed the prompt |
| there; the 0.5B text planner picks wrong actions; the accessibility-tree detector is not |
| implemented, so labels come from OCR only; and the harness still trusts a model's `done` |
| when no verifier is given. |
| |
| ## Roadmap |
| |
| - [x] Capture, OCR index, closed vocabulary, dry-run default, policy gates, doctor |
| - [x] A real click on a real window, with a verified outcome |
| - [x] Blind-grid-click guard, echo detection, no-progress guard |
| - [x] The harness suite running on real Windows (Server 2022 guest, no KVM — see above) |
| - [ ] Windows run of the *live desktop* test: the workflow is committed, but it needs a |
| GitHub mirror with Actions enabled — Hugging Face does not run GitHub Actions |
| - [ ] Make the OCR guard in `test_cli.py` skip, rather than fail, when no engine is installed |
| - [ ] Make the VLM reliable on a live screen (it echoes the prompt instead of choosing) |
| - [ ] Accessibility-tree detector (AT-SPI on Linux, UI Automation on Windows) |
| - [ ] Score episodes: replay a trace and count how often the choice was right |
| - [ ] marimo console: live screen preview, action log, pause / resume / step |
| |
| ## License |
| |
| MIT — see [LICENSE](LICENSE). |
| ''' |
|
|
| mo.md(readme_md) |
| return (readme_md,) |
|
|
|
|
| @app.cell(hide_code=True) |
| def repo_files(): |
| pyproject_toml = ''' |
| [build-system] |
| requires = ["setuptools>=68"] |
| build-backend = "setuptools.build_meta" |
| |
| [project] |
| name = "tiny-computer-agent" |
| version = "0.3.0" |
| description = "An agentic harness that drives a real computer with a tiny, fully local model - no cloud API." |
| readme = "README.md" |
| requires-python = ">=3.10" |
| license = { text = "MIT" } |
| authors = [{ name = "hkblockchaintechnology" }] |
| keywords = ["computer-use", "gui-agent", "agentic-harness", "local-first", "tiny-models"] |
| classifiers = [ |
| "Development Status :: 3 - Alpha", |
| "Intended Audience :: Developers", |
| "Programming Language :: Python :: 3 :: Only", |
| "Topic :: Scientific/Engineering :: Artificial Intelligence", |
| ] |
| dependencies = [] # the harness core imports nothing outside the standard library |
| |
| [project.optional-dependencies] |
| ocr = ["rapidocr-onnxruntime>=1.3"] # labels from pixels, no system binary |
| desktop = ["mss>=9.0", "pyautogui>=0.9", "pillow>=10"] # capture + real input |
| vision = ["transformers>=4.48", "torch>=2.2", "pillow>=10"] # tiny model brains |
| dev = ["pytest>=8"] |
| |
| [project.scripts] |
| tiny-computer-agent = "tiny_computer_agent.__main__:main" |
| |
| [project.urls] |
| Homepage = "https://huggingface.co/hkblockchaintechnology/tiny-computer-agent" |
| |
| [tool.setuptools] |
| packages = ["tiny_computer_agent"] |
| |
| [tool.pytest.ini_options] |
| testpaths = ["tests"] |
| ''' |
|
|
| license_txt = '''MIT License |
| |
| Copyright (c) 2026 hkblockchaintechnology |
| |
| Permission is hereby granted, free of charge, to any person obtaining a copy |
| of this software and associated documentation files (the "Software"), to deal |
| in the Software without restriction, including without limitation the rights |
| to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| copies of the Software, and to permit persons to whom the Software is |
| furnished to do so, subject to the following conditions: |
| |
| The above copyright notice and this permission notice shall be included in all |
| copies or substantial portions of the Software. |
| |
| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| SOFTWARE. |
| ''' |
|
|
| gitignore_txt = '''__pycache__/ |
| *.py[cod] |
| .venv/ |
| venv/ |
| .env |
| *.gguf |
| models/ |
| runs/ |
| traces/ |
| .ipynb_checkpoints/ |
| .DS_Store |
| ''' |
|
|
| mo.md(f"packaging: {len(pyproject_toml) + len(license_txt) + len(gitignore_txt)} chars, " |
| f"v0.2.0 with ocr/desktop/vision extras") |
| return gitignore_txt, license_txt, pyproject_toml |
|
|
|
|
| @app.cell |
| def publish(): |
| REPO_ID = "hkblockchaintechnology/tiny-computer-agent" |
|
|
| publish_button = mo.ui.button( |
| label="⬆️ Publish README.md to the Hub", |
| value=0, |
| on_click=lambda value: value + 1, |
| ) |
|
|
| publish_button |
| return REPO_ID, publish_button |
|
|
|
|
| @app.cell |
| def publish_status( |
| REPO_ID, |
| gitignore_txt, |
| license_txt, |
| publish_button, |
| pyproject_toml, |
| readme_md, |
| ): |
| _status = mo.md( |
| "This notebook owns the project: the card, the packaging files, the package and " |
| "the tests. Click **Publish** to push all of it to the Hub." |
| ).callout(kind="info") |
|
|
| if publish_button.value: |
| _token = os.environ.get("HF_TOKEN") |
| if not _token: |
| _status = mo.md( |
| "No `HF_TOKEN` in the environment. Run `export HF_TOKEN=...` " |
| "(or `hf auth login`), then restart the kernel." |
| ).callout(kind="warn") |
| else: |
| _api = HfApi(token=_token) |
| _root = Path(__file__).parent |
| _notebook = Path(__file__) |
| _files = { |
| "README.md": readme_md, |
| "pyproject.toml": pyproject_toml, |
| "LICENSE": license_txt, |
| ".gitignore": gitignore_txt, |
| "tiny_computer_agent_app.py": _notebook.read_text(encoding="utf-8"), |
| } |
| _folders = ["tiny_computer_agent", "tests"] |
| if (_root / ".github").is_dir(): |
| _folders.append(".github") |
| _missing = [f for f in ("tiny_computer_agent", "tests") if not (_root / f).is_dir()] |
| if _missing: |
| _status = mo.md(f"Nothing to publish: missing {_missing} next to `{_root}`").callout(kind="danger") |
| else: |
| try: |
| for _name, _text in _files.items(): |
| _api.upload_file( |
| path_or_fileobj=_text.encode("utf-8"), |
| path_in_repo=_name, |
| repo_id=REPO_ID, |
| repo_type="model", |
| commit_message=f"Publish {_name} from marimo", |
| ) |
| for _folder in _folders: |
| _api.upload_folder( |
| folder_path=str(_root / _folder), |
| path_in_repo=_folder, |
| repo_id=REPO_ID, |
| repo_type="model", |
| commit_message=f"Publish {_folder}/ from marimo", |
| ignore_patterns=["__pycache__/*", "*.pyc"], |
| ) |
| _status = mo.md( |
| f"✅ Published {len(_files)} files and {len(_folders)} folders to " |
| f"[{REPO_ID}](https://huggingface.co/{REPO_ID})" |
| ).callout(kind="success") |
| except Exception as _e: |
| _status = mo.md(f"Publish failed: `{type(_e).__name__}: {_e}`").callout(kind="danger") |
|
|
| _status |
| return |
|
|
|
|
| @app.cell |
| def demo(): |
| from tiny_computer_agent import AgentLoop, DryRunExecutor, PlannerBrain, Policy |
| from tiny_computer_agent.demo import fake_desktop |
| from tiny_computer_agent.observe import ImageScreen |
| from tiny_computer_agent.perception import IndexedScreen, OcrDetector |
|
|
| |
| |
| _detector = OcrDetector() |
| _observation = IndexedScreen(ImageScreen(fake_desktop()), _detector).capture() |
|
|
| perception_rows = [ |
| {"label": e.label, "kind": e.kind, "x": e.x, "y": e.y, "w": e.width, "h": e.height} |
| for e in _observation.elements |
| ] |
|
|
| perception_episode = AgentLoop( |
| screen=IndexedScreen(ImageScreen(fake_desktop()), _detector), |
| brain=PlannerBrain(), |
| executor=DryRunExecutor(), |
| policy=Policy(step_budget=4), |
| ).run("click \'Save\' to save the document") |
|
|
| _clicked = next((s.result.note for s in perception_episode.steps if s.action and s.action.kind == "click"), "n/a") |
|
|
| mo.vstack( |
| [ |
| mo.md( |
| f"**Perception, from pixels.** OCR read **{len(perception_rows)} labels** off a " |
| f"synthetic screenshot; the planner then chose one and the harness resolved it to a real pixel." |
| ), |
| mo.ui.table(perception_rows, selection=None), |
| mo.md(f"`{_clicked}`"), |
| mo.md(f"`{perception_episode.render().splitlines()[1]}`"), |
| ] |
| ) |
| return |
|
|
|
|
| if __name__ == "__main__": |
| app.run() |
|
|
